package ru.yandex.wmconsole.periodic;

import java.net.URL;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Calendar;
import java.util.Date;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Required;
import org.springframework.jdbc.core.simple.ParameterizedRowMapper;

import ru.yandex.common.scheduler.ExecutionContext;
import ru.yandex.wmconsole.data.AddUrlRequest;
import ru.yandex.wmconsole.data.HostInfoStatusEnum;
import ru.yandex.wmconsole.data.partition.WMCPartition;
import ru.yandex.wmconsole.service.AddUrlService;
import ru.yandex.wmtools.common.error.InternalException;
import ru.yandex.wmtools.common.error.UserException;
import ru.yandex.wmtools.common.servantlet.AbstractServantlet;
import ru.yandex.wmtools.common.util.SqlUtil;
import ru.yandex.wmtools.common.util.URLUtil;
import ru.yandex.wmtools.common.util.scheduler.timetable.AbstractLockableTaskExecutor;

/**
 * Добавляет подтвержденные хосты со статусом WAITING или ROBOTS_TXT в addurl
 *
 * User: azakharov
 * Date: 20.07.12
 * Time: 16:38
 */
public class AddUrlTask  extends AbstractLockableTaskExecutor {
    private static final Logger log = LoggerFactory.getLogger(AddUrlTask.class);

    private static final String FIELD_NAME = "name";

    private int sleepPeriodBeforeRetryMillis = 60000; // 60 сек
    private int sleepPeriodBetweenAddurlMillis = 60; // 60 мс
    private int maxAttempts = 3;

    private static final String ADD_URL_LOCK_NAME = "add_url_task_lock";

    private static final String SELECT_VERIFIED_HOSTS_COUNT_QUERY =
            "SELECT " +
                    "   COUNT(DISTINCT host_id) " +
                    "FROM " +
                    "   tbl_users_hosts " +
                    "WHERE " +
                    "   state = 1 " +
                    "   AND verified_on <= DATE_SUB(?, INTERVAL 1 WEEK)";

    private static final int BATCH_SIZE = 10240;

    private static final String SELECT_VERIFIED_HOSTS_QUERY =
            "SELECT name AS " + FIELD_NAME + " FROM tbl_hosts h " +
                    "WHERE EXISTS (SELECT uh.host_id FROM tbl_users_hosts uh " +
                    "              WHERE uh.host_id = h.host_id AND state = 1 AND verified_on <= DATE_SUB(?, INTERVAL 1 WEEK)) " +
                    "ORDER BY h.host_id " +
                    "LIMIT ?, " + BATCH_SIZE;

    private static final String SELECT_WAITING_OR_ROBOTS_TXT_HOSTS_QUERY =
            "SELECT " +
                    "  name AS " + FIELD_NAME + " " +
                    "FROM " +
                    "   tbl_hosts h " +
                    "JOIN tbl_host_info hi " +
                    "ON h.host_id = hi.host_id " +
                    "LEFT JOIN tbl_links_cache lc " +
                    "ON hi.host_id = lc.host_id " +
                    "WHERE " +
                    "   (hi.status = " + HostInfoStatusEnum.WAITING.getValue() +
                    "   OR hi.status = " + HostInfoStatusEnum.ROBOTS_TXT.getValue() + ") " +
                    "   AND (lc.index_count IS NULL OR lc.index_count = 0) " +
                    "   AND h.name IN (%1$s)";

    private static final SqlUtil.ListParameterizer<String> PARAMETERIZER = new SqlUtil.ListParameterizer<String>() {
        @Override
        public String getParameter(int i, String obj) {
            return obj;
        }

        @Override
        public int getParamNumber() {
            // скобочки будут в запросе
            return 1;
        }

        @Override
        public boolean isQuotesNeeded(int i) {
            return true;
        }
    };

    private static final ParameterizedRowMapper<String> rowMapper = new ParameterizedRowMapper<String>() {
        @Override
        public String mapRow(ResultSet resultSet, int i) throws SQLException {
            return resultSet.getString(FIELD_NAME);
        }
    };

    private AddUrlService addUrlService;

    @Override
    public String runWithRELogging(ExecutionContext context) {
        long ms1 = Calendar.getInstance().getTimeInMillis();
        log.debug("Add url task triggered");
        logUserDbConnections();
        Date curDate = new Date();
        if (!getLock(WMCPartition.nullPartition(), ADD_URL_LOCK_NAME, 3 * 24 * 60)) {
            String msg = "Failed to get lock for addurl task";
            log.warn(msg);
            return msg;
        }

        try {
            long verifiedHostsCount = getJdbcTemplate(WMCPartition.nullPartition()).queryForLong(
                    SELECT_VERIFIED_HOSTS_COUNT_QUERY, curDate);
            long startIndex = 0;

            final int hostDbCount = WMCPartition.getHostDbCount(getDatabaseCount());

            final Map<Integer, List<String>> hostDbQueues = new LinkedHashMap<Integer, List<String>>();

            // Число неуспешных попыток чтения из пользовательской БД
            int userDbAttempts = 0;

            while (startIndex < verifiedHostsCount) {
                // Получаем очередную пачку хостов из пользовательской БД
                final List<String> hosts;
                try {
                    hosts = getJdbcTemplate(WMCPartition.nullPartition()).query(
                            SELECT_VERIFIED_HOSTS_QUERY, rowMapper, curDate, startIndex);
                    startIndex = startIndex + BATCH_SIZE;
                    userDbAttempts = 0;
                } catch (InternalException e) {
                    log.debug("Exception while getting verified hosts. Sleeping  " + sleepPeriodBeforeRetryMillis, e);
                    userDbAttempts++;
                    if (userDbAttempts > maxAttempts) {
                        final String s = "Too many attempts to read userdb - stopping task";
                        log.error(s);
                        return s;
                    }
                    try {
                        Thread.sleep(sleepPeriodBeforeRetryMillis);
                    } catch (InterruptedException e2) {
                        log.error("Exception while sleep ", e2);
                    }
                    continue;
                }

                hostDbQueues.clear();
                for (int i = 0 ; i < hostDbCount; i++) {
                    hostDbQueues.put(i, new LinkedList<String>());
                }

                // Распределяем хосты по хостовым базам
                for (String host : hosts) {
                    if (URLUtil.isSpamerDomain(host)) {
                        // не добавляем спамерские сайты в addUrl
                        continue;
                    }
                    int dbIndex = WMCPartition.getDatabaseIndex(getDatabaseCount(), host);
                    List<String> queue = hostDbQueues.get(dbIndex);
                    queue.add(host);
                }

                // Делаем запросы к хостовым базам
                for (int i = 0 ; i < hostDbCount; i++) {
                    logConnections(i);
                    int hostDbAttempts = 0;

                    // Получаем список хостов, которые относятся к данной хостовой базе
                    final List<String> hs = hostDbQueues.get(i);
                    if (hs.isEmpty()) {
                        continue;
                    }

                    List<String> toUpdate;
                    try {
                        // Получаем хосты, у которых статус WAITING или ROBOTS_TXT
                        toUpdate = getJdbcTemplate(new WMCPartition(i)).query(
                                String.format(SELECT_WAITING_OR_ROBOTS_TXT_HOSTS_QUERY, SqlUtil.getCommaSeparatedList(hs, PARAMETERIZER)),
                                rowMapper);
                        hostDbAttempts = 0;
                    } catch (InternalException e) {
                        log.error("Can't get hosts for addurl from host db", e);
                        hostDbAttempts++;
                        if (hostDbAttempts > maxAttempts) {
                            final String s = "Too many attempts to read hostdb "+ i + " - stopping task";
                            log.error(s);
                            logConnections(i);
                            return s;
                        }
                        try {
                            Thread.sleep(sleepPeriodBeforeRetryMillis);
                        } catch (InterruptedException e2) {
                            log.error("Exception while sleep ", e2);
                        }
                        continue;
                    }

                    // Добавляем эти хосты в файл addurl для отправки роботу
                    for (String host : toUpdate) {
                        log.debug("ADDING_HOST_TO_ADDURL " + host);
                        URL url = AbstractServantlet.prepareUrl(host, true);
                        try {
                            // Делаем sleep, чтобы не заваливать запросами базу и xml-поиск
                            try {
                                Thread.sleep(sleepPeriodBetweenAddurlMillis);
                            } catch (InterruptedException e) {
                                //ignore
                               log.error("Exception while sleep", e);
                            }
                            // Указываем домен "ru" сознательно, так как для большинства сайтов это верно
                            AddUrlRequest addUrlRequest = new AddUrlRequest(url, null, 0L, null, null);
                            addUrlService.addUrl(addUrlRequest, false, "ru", true);
                            log.debug("Url " + url + " was successfully added");
                        } catch (UserException e) {
                            log.info("can't add host to addurl " + e.getMessage());
                        } catch (InternalException ie) {
                            log.error("can't add host to addurl " + ie.getMessage());
                        }
                    }

                    logConnections(i);
                }
            }
            long ms2 = Calendar.getInstance().getTimeInMillis();
            log.info("Processing time of addurl task is " + (ms2 - ms1));
        } catch (Exception e) {
            log.error("Exception in addurl task " + e.getMessage(), e);
            long ms2 = Calendar.getInstance().getTimeInMillis();
            log.info("Processing time of addurl task is " + (ms2 - ms1));
        } finally {
            releaseLock(WMCPartition.nullPartition(), ADD_URL_LOCK_NAME);
        }
        return null;
    }

    @Required
    public void setAddUrlService(AddUrlService addUrlService) {
        this.addUrlService = addUrlService;
    }

    @Required
    public void setSleepPeriodBeforeRetryMillis(int sleepPeriodBeforeRetryMillis) {
        this.sleepPeriodBeforeRetryMillis = sleepPeriodBeforeRetryMillis;
    }

    @Required
    public void setSleepPeriodBetweenAddurlMillis(int sleepPeriodBetweenAddurlMillis) {
        this.sleepPeriodBetweenAddurlMillis = sleepPeriodBetweenAddurlMillis;
    }

    @Required
    public void setMaxAttempts(int maxAttempts) {
        this.maxAttempts = maxAttempts;
    }
}
