package ru.yandex.wmconsole.service;

import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;

import org.jetbrains.annotations.Nullable;
import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Required;
import org.springframework.transaction.TransactionStatus;

import ru.yandex.common.util.collections.Pair;
import ru.yandex.common.util.concurrent.CommonThreadFactory;
import ru.yandex.common.util.db.LongRowMapper;
import ru.yandex.webmaster.common.host.dao.TblUsersHostsDao;
import ru.yandex.webmaster.common.urltree.YandexSearchShard;
import ru.yandex.wmconsole.data.IndexingEntityEnum;
import ru.yandex.wmconsole.data.Node;
import ru.yandex.wmconsole.data.VerificationStateEnum;
import ru.yandex.wmconsole.data.info.BriefHostInfo;
import ru.yandex.wmconsole.data.info.HostDbHostInfo;
import ru.yandex.wmconsole.data.info.TreeInfo;
import ru.yandex.wmconsole.data.partition.WMCPartition;
import ru.yandex.wmconsole.service.dao.TblPeriodicCounterDao;
import ru.yandex.wmconsole.service.dao.TblUrlTreesDao;
import ru.yandex.wmtools.common.data.xmlsearch.XmlSearchRequest;
import ru.yandex.wmtools.common.error.InternalException;
import ru.yandex.wmtools.common.error.UserException;
import ru.yandex.wmtools.common.servantlet.AbstractServantlet;
import ru.yandex.wmtools.common.service.AbstractDbService;
import ru.yandex.wmtools.common.service.IndexInfoService;
import ru.yandex.wmtools.common.util.ParameterizedMapRowMapper;
import ru.yandex.wmtools.common.util.ServiceTransactionCallbackWithoutResult;
import ru.yandex.wmtools.common.util.SqlUtil;

/**
 * Сервис сохранения истории изменений ТИЦ, страниц в поиске, внешних ссылок
 *
 * User: azakharov
 * Date: 26.06.12
 * Time: 13:17
 */
public class IndexHistoryService extends AbstractDbService {
    private static final Logger log = LoggerFactory.getLogger(IndexHistoryService.class);

    private static final String INSERT_SAVE_HISTORY_QUERY =
            "INSERT INTO tbl_trend_%1$s (host_id, time, %1$s) VALUES (?, ?, ?)";

    private static final String INSERT_SAVE_HISTORY_FOR_SHARD_QUERY =
            "INSERT INTO tbl_trend_%1$s (host_id, time, %1$s, shard_id) VALUES (?, ?, ?, ?)";

    private static final String DELETE_OLD_DATA_QUERY =
            "DELETE FROM tbl_trend_%1$s WHERE host_id = ? AND time < (? - INTERVAL 3 MONTH)";

    private static final String INSERT_HISTORY_TEMPLATE_QUERY =
            "INSERT IGNORE INTO " +
                    "   tbl_trend_%1$s " +
                    "(host_id, time, %2$s)" +
                    "VALUES ";

    private static final String INSERT_HISTORY_WITH_SHARDS_TEMPLATE_QUERY =
            "INSERT IGNORE INTO " +
                    "   tbl_trend_%1$s " +
                    "(host_id, time, %2$s, shard_id)" +
                    "VALUES ";

    private static final String INSERT_HISTORY_VALUES = "(?, ?, ?)";
    private static final String INSERT_HISTORY_WITH_SHARDS_VALUES = "(?, ?, ?, ?)";

    private static final String FIELD_TIME = "time";
    private static final String FIELD_COUNT = "count";
    private static final String FIELD_SHARD_ID = "shard_id";

    private static final String SELECT_ERRORS_COUNT_HISTORY_QUERY =
            "SELECT " +
                    "   time AS " + FIELD_TIME + ", " +
                    "   sum(count) AS " + FIELD_COUNT + " " +
                    "FROM tbl_trend_code_error_trees_count " +
                    "WHERE " +
                    "   host_id = ? AND " +
                    "   code IN (%s) AND " +
                    "   time > ? " +
                    "GROUP BY time " +
                    "ORDER BY time ASC";

    private static final String SELECT_HISTORY_POINT_COUNT_QUERY =
            "SELECT COUNT(DISTINCT time) FROM tbl_trend_%1$s WHERE host_id = ?";

    private static final String SELECT_HISTORY_FOR_HOST_QUERY =
            "SELECT time, %1$s FROM tbl_trend_%2$s WHERE host_id = ?";

    private static final String SELECT_HISTORY_FOR_HOST_AND_SHARD_QUERY =
            "SELECT time, %1$s FROM tbl_trend_%2$s WHERE host_id = ? AND shard_id = ?";

    private static final String SELECT_HISTORY_FOR_HOST_AND_NODE_QUERY =
            "SELECT " +
                    "   time, " +
                    "   t.%1$s " +
                    "FROM " +
                    "   tbl_trend_%2$s t LEFT JOIN tbl_url_trees u ON t.node_id = u.id "+
                    "WHERE " +
                    "   host_id = ? AND node_id = ?";

    private static final String SELECT_HISTORY_FOR_HOST_AND_NODE_AND_SHARD_QUERY =
            "SELECT " +
                    "   time, " +
                    "   t.%1$s " +
                    "FROM " +
                    "   tbl_trend_%2$s t LEFT JOIN tbl_url_trees u ON t.node_id = u.id "+
                    "WHERE " +
                    "   host_id = ? AND node_id = ? AND shard_id = ?";

    private static final String GET_HISTORY_FOR_HOST_QUERY =
            "SELECT host_id, time, %1$s FROM tbl_trend_%2$s WHERE host_id = ?";

    private static final String GET_HISTORY_FOR_HOST_WITH_SHARDS_QUERY =
            "SELECT host_id, time, %1$s, shard_id FROM tbl_trend_%2$s WHERE host_id = ?";

    private static final int MAX_QUERY_SIZE = 1000000;

    private static final int BATCH_SIZE = 1024;
    // for batch of 1024 host approximate processing time is 10 minutes
    private static final int MAX_BATCH_PROCESSING_MINUTES = 20;

    private HostDbHostInfoService hostDbHostInfoService;
    private HostInfoService hostInfoService;
    private LinksCacheService linksCacheService;
    private UrlTreeService urlTreeService;

    private TblUsersHostsDao tblUsersHostsDao;

    private TblPeriodicCounterDao tblPeriodicCounterDao;
    private TblUrlTreesDao tblUrlTreesDao;

    // Предоставляют данные для последней точки на графике
    private Map<IndexingEntityEnum, ValueProvider> readableProviders = new HashMap<IndexingEntityEnum, ValueProvider>();
    // Предоставляют данные для сохранения в таблицы истории
    private Map<IndexingEntityEnum, ValueProvider> writableProviders = new HashMap<IndexingEntityEnum, ValueProvider>();

    private ThreadFactory threadFactory;
    private int historySaverThreadCount = 8;

    public void init() {
        threadFactory = new CommonThreadFactory(true, IndexHistoryService.class.getSimpleName() + "-");

        writableProviders.put(IndexingEntityEnum.TCY, new TcyProvider());
        writableProviders.put(IndexingEntityEnum.LINKS_COUNT, new ExternalLinksCountProvider());

        readableProviders.put(IndexingEntityEnum.TCY, new TcyProvider());
        readableProviders.put(IndexingEntityEnum.LINKS_COUNT, new ExternalLinksCountProvider());
        readableProviders.put(IndexingEntityEnum.URL_TREES_INDEX_COUNT, new UrlTreesIndexCountProvider());
        readableProviders.put(IndexingEntityEnum.URL_TREES_URLS, new UrlTreesUrlsProvider());
    }

    private void saveValue(HostDbHostInfo hostDbHostInfo, long value, DateTime date, String name) {
        try {
            int dbIndex = WMCPartition.getDatabaseIndex(getDatabaseCount(), hostDbHostInfo.getName());
            logConnections(dbIndex);
            getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).update(
                    String.format(INSERT_SAVE_HISTORY_QUERY, name),
                    hostDbHostInfo.getHostDbHostId(), date.toDate(), value);
        } catch (InternalException ie) {
            log.error(" exception while saving " + name + " for host_id " +
                    hostDbHostInfo.getHostDbHostId() + " value " + value, ie);
        }
    }

    private void saveValueForShard(HostDbHostInfo hostDbHostInfo, long value, DateTime date, String name, YandexSearchShard searchShard) {
        try {
            int dbIndex = WMCPartition.getDatabaseIndex(getDatabaseCount(), hostDbHostInfo.getName());
            logConnections(dbIndex);
            getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).update(
                    String.format(INSERT_SAVE_HISTORY_FOR_SHARD_QUERY, name),
                    hostDbHostInfo.getHostDbHostId(), date.toDate(), value, searchShard.value());
        } catch (InternalException ie) {
            log.error(" exception while saving " + name + " for host_id " +
                    hostDbHostInfo.getHostDbHostId() + " value " + value, ie);
        }
    }

    private void removeOldValues(HostDbHostInfo hostDbHostInfo, DateTime date, String name) throws InternalException {
        try {
            String query = String.format(DELETE_OLD_DATA_QUERY, name);
            getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).update(
                    query,
                    hostDbHostInfo.getHostDbHostId(), date.toDate()
            );
            int dbIndex = WMCPartition.getDatabaseIndex(getDatabaseCount(), hostDbHostInfo.getName());
            logConnections(dbIndex);
        } catch (InternalException ie) {
            log.error(" exception while removing old values " + name +
                    " for host_id " + hostDbHostInfo.getHostDbHostId(), ie);
        }
    }

    private Long getTaskBatchSize(long maxVerifiedHostId) throws InternalException {
        return (long)((maxVerifiedHostId / (24 * 2 * 4)) * 1.5);
    }

    private Pair<Long, Long> getHostIdRage(long maxVerifiedHostId) throws InternalException {
        long minHostId = tblPeriodicCounterDao.getIndexHistoryTaskHostId();
        long maxHostId = minHostId + getTaskBatchSize(maxVerifiedHostId);

        return new Pair<Long, Long>(minHostId, maxHostId);
    }

    private void updateMinHostId(Pair<Long, Long> currentRange, long maxVerifiedHostId) throws InternalException {
        long nextMinHostId = (currentRange.getSecond() + 1 <= maxVerifiedHostId) ? (currentRange.getSecond() + 1) : 0;
        tblPeriodicCounterDao.updateIndexHistoryTaskHostId(nextMinHostId);
    }

    public void saveIndexHistory(IndexHistoryAliveHandler handler) throws InternalException {
        DateTime now = DateTime.now();
        log.info("Start index history saver");
        logUserDbConnections();

        long maxVerifiedHostId = tblUsersHostsDao.getMaxVerifiedHostId(now);
        Pair<Long,Long> range = getHostIdRage(maxVerifiedHostId);
        long minHostId = range.getFirst();
        long maxHostId = range.getSecond();

        long hostCount = tblUsersHostsDao.countDistinctVerifiedHosts(minHostId, maxHostId);
        long maxBatchId = hostCount / BATCH_SIZE + 1;
        AtomicLong batchCounter = new AtomicLong(0);
        log.info("Update index history for {} hosts", hostCount);
        ExecutorService executorService = Executors.newFixedThreadPool(historySaverThreadCount, threadFactory);
        long startTimeNanos = System.nanoTime();
        for (int i = 0; i < historySaverThreadCount; i++) {
            executorService.submit(new HistoryUpdateRunnable(batchCounter, maxBatchId, now, minHostId, maxHostId));
        }
        try {
            long lastBatch = 0;
            long currentBatch;
            while((currentBatch = batchCounter.get()) < maxBatchId) {
                Thread.sleep(5000);
                if (executorService.isTerminated()) {
                    log.error("All save threads terminated unexpectedly on batch {} of {}", currentBatch, maxBatchId);
                    break;
                }
                if (currentBatch > 0 && currentBatch - lastBatch > 5) {
                    long nowNanos = System.nanoTime();
                    long timeRemainNanos = (long) ((nowNanos - startTimeNanos) * (1.0f * maxBatchId / currentBatch - 1.0f));
                    log.info("Batch {} of {} finished. Time spent {} s, time remain {} s",
                            currentBatch, maxBatchId,
                            TimeUnit.NANOSECONDS.toSeconds(nowNanos - startTimeNanos),
                            TimeUnit.NANOSECONDS.toSeconds(timeRemainNanos)
                    );
                    lastBatch = currentBatch;
                }
                try {
                    handler.handle();
                } catch (Exception e) {
                    log.error("exception in index history alive handler", e);
                }
            }
            // schedule next range
            updateMinHostId(range, maxVerifiedHostId);

            executorService.shutdown();
            executorService.awaitTermination(MAX_BATCH_PROCESSING_MINUTES, TimeUnit.MINUTES);
            if (!executorService.isTerminated()) {
                executorService.shutdownNow();
            }
        } catch (InterruptedException e) {
            log.error("Update was interrupted, stop", e);
            executorService.shutdownNow();
        }
        logUserDbConnections();
        log.info("Index history saver stopped. Time spent {} s",
                TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - startTimeNanos));
    }

    private class HistoryUpdateRunnable implements Runnable {
        private final DateTime now;
        private final AtomicLong batchIdCounter;
        private final long maxBatchId;
        private final long minHostId;
        private final long maxHostId;

        private HistoryUpdateRunnable(AtomicLong batchIdCounter, long maxBatchId, DateTime now, long minHostId, long maxHostId) {
            this.batchIdCounter = batchIdCounter;
            this.maxBatchId = maxBatchId;
            this.now = now;
            this.minHostId = minHostId;
            this.maxHostId = maxHostId;
        }

        public List<Long> getBatch(Long minHostId, Long maxHostId, long batchId) throws InternalException {
            final String queryTemplate =
                    "SELECT DISTINCT host_id FROM "+
                    "       tbl_users_hosts uh " +
                    "WHERE "+
                    "   uh.host_id >= ? AND uh.host_id <= ? " +
                    "AND " +
                    "   uh.state in (%s) " +
                    "AND " +
                    "   uh.verified_on < ? " +
                    "ORDER BY uh.host_id " +
                    "LIMIT %d, %d";
            String query = String.format(queryTemplate,
                    VerificationStateEnum.getCommaSeparatedListForVerifiedStates(),
                    batchId * BATCH_SIZE, BATCH_SIZE);
            List<Long> hostIds = getJdbcTemplate(WMCPartition.nullPartition()).query(
                    query, new LongRowMapper(), minHostId, maxHostId, now.toDate());
            return hostIds;
        }

        @Override
        public void run() {
            log.info("Start history update thread");
            long batchId;
            while ((batchId = batchIdCounter.getAndIncrement()) < maxBatchId) {
                log.info("Start batch: {}", batchId);
                if (Thread.interrupted()) {
                    log.warn("Batch {} was interrupted, stop", batchId);
                    return;
                }
                try {
                    List<Long> hostIds = getBatch(minHostId, maxHostId, batchId);
                    for (Long hostId : hostIds) {
                        if (Thread.interrupted()) {
                            log.warn("Batch {} was interrupted, stop", batchId);
                            return;
                        }
                        log.debug("Update host: {} in batch {}", hostId, batchId);

                        String hostName = hostInfoService.getHostNameByHostId(hostId);
                        if (hostName == null) {
                            continue;
                        }
                        HostDbHostInfo hostDbHostInfo = hostDbHostInfoService.getHostDbHostInfo(hostName);

                        for (Map.Entry<IndexingEntityEnum, ValueProvider> entry : writableProviders.entrySet()) {
                            String entity = entry.getKey().getName();
                            ValueProvider provider = entry.getValue();

                            if (entry.getKey().isEntityForSearchShard()) {

                                for (YandexSearchShard s : YandexSearchShard.values()) {
                                    Long value = provider.provideValue(hostDbHostInfo, null, s);
                                    log.debug(entity + " " + hostDbHostInfo.getHostDbHostId() + " " + value);
                                    if (value != null) {
                                        saveValueForShard(hostDbHostInfo, value, now, entity, s);
                                        removeOldValues(hostDbHostInfo, now, entity);
                                    }
                                }

                            } else {
                                Long value = provider.provideValue(hostDbHostInfo, null, null);
                                log.debug(entity + " " + hostDbHostInfo.getHostDbHostId() + " " + value);
                                if (value != null) {
                                    saveValue(hostDbHostInfo, value, now, entity);
                                    removeOldValues(hostDbHostInfo, now, entity);
                                }
                            }
                        }
                    }
                } catch (InternalException e) {
                    log.error("Unable to process batch " + batchId + ", skip", e);
                }
            }
            log.info("Stop history update thread");
        }
    }

    public NavigableMap<Date, Long> getErrorsHistoryForTrend(
            final HostDbHostInfo hostDbHostInfo,
            final List<Integer> codes,
            final Date date) throws InternalException {
        return getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).queryForNavigableMap(
                String.format(SELECT_ERRORS_COUNT_HISTORY_QUERY, SqlUtil.getCommaSeparatedList(codes)),
                errorHistoryForTrendRowMapper,
                hostDbHostInfo.getHostDbHostId(),
                date
                );
    }

    public static Long calculateTrend(NavigableMap<Date, Long> history) {
        final long borderValue = 100;       // Для всех, кроме tcy минимальное значение 100
        final double borderPercent = 0.15d; // Для всех, кроме index_count пороговое значение 15%

        if (history.size() <= 1) {
            return null;
        }

        final Map.Entry<Date, Long> lastEntry = history.lastEntry();
        final Date maxDate = lastEntry.getKey();
        final Long lastVal = lastEntry.getValue();
        Calendar c = Calendar.getInstance();
        c.setTime(maxDate);
        c.add(Calendar.DATE, -3);
        final Date prevDate = c.getTime();
        Long prevVal = lastVal;

        for (Map.Entry<Date, Long> entry : history.entrySet()) {
            if (entry.getKey().after(prevDate)) {
                break;
            }
            prevVal = entry.getValue();
        }
        if (prevVal < borderValue && lastVal < borderValue) {
            return null;
        }

        long trend = lastVal - prevVal;

        if ((double)lastVal >= (double)prevVal * (1.0d + borderPercent)) {
            return trend;
        }
        if ((double)lastVal <= (double)prevVal * (1.0d - borderPercent)) {
            return trend;
        }
        return 0L;
    }

    /**
     * Копирует данные истории показателей индексирования для хоста при переклейке зеркал
     *
     * @param oldHostDbHostInfo информация о старом хосте
     * @param newHostDbHostInfo информация о новом хосте
     * @throws InternalException
     */
    public void tryToCopyHistoryData(final HostDbHostInfo oldHostDbHostInfo, final HostDbHostInfo newHostDbHostInfo) throws InternalException, UserException {
        if (!hasHistory(newHostDbHostInfo)) {
            // Формируем список запросов на вставку данных в таблицу истории
            final List<Pair<String, Object[]>> queries = new LinkedList<Pair<String, Object[]>>();
            for (IndexingEntityEnum entity : IndexingEntityEnum.getEntitiesForHost()) {
                queries.addAll(copyHistoryFor(oldHostDbHostInfo, newHostDbHostInfo, entity));
            }

            getServiceTransactionTemplate(new WMCPartition(newHostDbHostInfo, null))
                    .executeInService(new ServiceTransactionCallbackWithoutResult() {
                        @Override
                        protected void doInTransactionWithoutResult(TransactionStatus transactionStatus) throws UserException, InternalException {
                            log.debug("Applying " + queries.size() + " queries");
                            for (Pair<String, Object[]> query : queries) {
                                getJdbcTemplate(new WMCPartition(newHostDbHostInfo, null))
                                        .update(query.first, query.second);
                            }
                        }
                    });
            log.debug("History data have been copied successfully");
        } else {
            log.debug("Already have history data for host " + newHostDbHostInfo);
        }
    }

    /**
     * Получает данные истории одного показателя в виде запросов на вставку
     *
     * @param oldHostDbHostInfo информация о старом хосте в хостовой базе
     * @param newHostDbHostInfo информация о новом хосте в хостовой базе
     * @param entity            индексируемый показатель
     * @return                  список пар из SQL-запроса и параметров запроса
     */
    List<Pair<String, Object[]>> copyHistoryFor(
            HostDbHostInfo oldHostDbHostInfo,
            HostDbHostInfo newHostDbHostInfo,
            IndexingEntityEnum entity) throws InternalException {

        String template = entity.isEntityForSearchShard() ? GET_HISTORY_FOR_HOST_WITH_SHARDS_QUERY : GET_HISTORY_FOR_HOST_QUERY;
        List<Map<String, Object>> params = getJdbcTemplate(new WMCPartition(oldHostDbHostInfo, null))
                 .queryForList(
                         String.format(template, entity.getColumn(), entity.getName()),
                         oldHostDbHostInfo.getHostDbHostId());

        List<Object> pars = new LinkedList<Object>();
        StringBuilder q = new StringBuilder();
        List<Pair<String, Object[]>> res = new LinkedList<Pair<String, Object[]>>();
        String insertTemplate = entity.isEntityForSearchShard() ? INSERT_HISTORY_WITH_SHARDS_TEMPLATE_QUERY : INSERT_HISTORY_TEMPLATE_QUERY;
        final String insertQuery = String.format(insertTemplate, entity.getName(), entity.getColumn());
        for (Map<String, Object> row : params) {
            pars.add(newHostDbHostInfo.getHostDbHostId());
            pars.add(row.get(FIELD_TIME));
            pars.add(row.get(entity.getColumn()));
            if (entity.isEntityForSearchShard()) {
                pars.add(row.get(FIELD_SHARD_ID));
            }

            if (q.length() == 0) {
                q.append(insertQuery);
            } else {
                q.append(",");
            }
            if (entity.isEntityForSearchShard()) {
                q.append(INSERT_HISTORY_WITH_SHARDS_VALUES);
            } else {
                q.append(INSERT_HISTORY_VALUES);
            }

            if (q.length() > MAX_QUERY_SIZE) {
                res.add(Pair.of(q.toString(), pars.toArray()));
                q = new StringBuilder();
                pars.clear();
            }
        }

        if (!pars.isEmpty()) {
            res.add(Pair.of(q.toString(), pars.toArray()));
        }

        return res;
    }

    /**
     * Проверяет наличие истории для сайта
     *
     * @param hostDbHostInfo    информация о хосте в хостовой базе
     * @return                  имеются ли данные в таблице истории для сайта
     * @throws InternalException
     */
    private boolean hasHistory(HostDbHostInfo hostDbHostInfo) throws InternalException {
        for (IndexingEntityEnum entity : IndexingEntityEnum.getEntitiesForHost()) {
            // Проверяем, что показатель рассчитывается лоадером, так как для показателей, рассчитываемых в java может
            // оказаться больше точек для нового хоста
            if (entity.isFromLoader()) {
                // Проверяем на всякий случай, в истории для всех показателей, на случай неполного апдейта
                boolean entityHasHistory = hasHistoryFor(hostDbHostInfo, entity);
                if (entityHasHistory) {
                    return true;
                }
            }
        }
        return false;
    }

    /**
     * Проверяет наличие истории для показателя индексирования
     *
     * @param hostDbHostInfo    информация о хосте в хостовой базе
     * @param indexingEntity    показатель индексирования
     * @return                  есть ли данные в таблице истории для показателя индексирования
     * @throws InternalException
     */
    private boolean hasHistoryFor(HostDbHostInfo hostDbHostInfo, IndexingEntityEnum indexingEntity) throws InternalException {
        // Определяем количество точек истории для хоста
        long numberOfPoints = getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).queryForLong(
                String.format(
                        SELECT_HISTORY_POINT_COUNT_QUERY, indexingEntity.getName()),
                hostDbHostInfo.getHostDbHostId()
        );
        // Если больше одной точки в истории, то считаем, что история есть
        return numberOfPoints > 1;
    }

    /**
     * Получение истории индексирования показателя для отображения на графике
     *
     * @param briefHostInfo  информация о хосте
     * @param entity         показатель индексирования
     * @param node           идентификатор узла дерева в структуре сайта
     * @return               map из даты в количественное значение показателя
     * @throws InternalException
     */
    public NavigableMap<Date, Long> getIndexHistoryPlotData(
            final BriefHostInfo briefHostInfo, final IndexingEntityEnum entity, @Nullable final Long node, final YandexSearchShard searchShard) throws InternalException {
        HostDbHostInfo hostDbHostInfo = hostDbHostInfoService.getHostDbHostInfo(briefHostInfo.getName());
        IndexHistoryMapper mapper = new IndexHistoryMapper(entity.getColumn());
        final String query;

        final NavigableMap<Date, Long> result;
        if (entity.isEntityForNode()) {
            if (entity.isEntityForSearchShard()) {
                query = String.format(SELECT_HISTORY_FOR_HOST_AND_NODE_AND_SHARD_QUERY, entity.getColumn(), entity.getName());
                result = getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).queryForNavigableMap(
                        query, mapper, hostDbHostInfo.getHostDbHostId(), node, searchShard.value());
            } else {
                query = String.format(SELECT_HISTORY_FOR_HOST_AND_NODE_QUERY, entity.getColumn(), entity.getName());
                result = getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).queryForNavigableMap(
                        query, mapper, hostDbHostInfo.getHostDbHostId(), node);
            }
        } else {
            if (entity.isEntityForSearchShard()) {
                query = String.format(SELECT_HISTORY_FOR_HOST_AND_SHARD_QUERY, entity.getColumn(), entity.getName());
                result = getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).queryForNavigableMap(
                        query, mapper, hostDbHostInfo.getHostDbHostId(), searchShard.value());
            } else {
                query = String.format(SELECT_HISTORY_FOR_HOST_QUERY, entity.getColumn(), entity.getName());
                result = getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).queryForNavigableMap(
                        query, mapper, hostDbHostInfo.getHostDbHostId());
            }
        }

        if (!result.isEmpty()) {
            ValueProvider provider = readableProviders.get(entity);
            DateTime dateTime = new DateTime();
            Date current = dateTime.toDateMidnight().toDate();
            if (provider != null) {
                result.put(current, provider.provideValue(hostDbHostInfo, node, searchShard));
            } else {
                result.put(current, result.lastEntry().getValue());
            }
        }

        return result;
    }

    public static abstract class ValueProvider {
        public abstract Long provideValue(HostDbHostInfo hostInfo, Long nodeId, YandexSearchShard searchShard);
    }

    public class TcyProvider extends ValueProvider {
        @Override
        public Long provideValue(final HostDbHostInfo hostInfo, final Long nodeId, final YandexSearchShard searchShard) {
            try {
                Integer tcy = linksCacheService.checkCacheAndGetTcy(
                        AbstractServantlet.prepareUrl(hostInfo.getName(), true));
                return tcy != null ? tcy.longValue() : null;
            } catch (InternalException e) {
                log.error("Exception while getting tcy ", e);
                return null;
            } catch (UserException e) {
                log.error("Exception while getting tcy ", e);
                return null;
            }
        }
    }

    public class UrlTreesIndexCountProvider extends ValueProvider {
        @Override
        public Long provideValue(final HostDbHostInfo hostInfo, final Long nodeId, final YandexSearchShard searchShard) {
            try {
                YandexSearchShard shard = tblUrlTreesDao.getOptimumShardId(hostInfo);
                Long result = tblUrlTreesDao.getIndexCount(hostInfo, nodeId, shard);
                if (result != null) {
                    return result;
                } else {
                    TreeInfo treeInfo = urlTreeService.getUrlTreeInfo(hostInfo, nodeId, true, YandexSearchShard.RU);
                    Node n = treeInfo.getId2node().get(nodeId);
                    if (n != null && n.getInfo() != null) {
                        return n.getInfo().getIndexCount();
                    } else {
                        return null;
                    }
                }
            } catch (InternalException e) {
                log.error("Exception while getting index_count ", e);
                return null;
            } catch (UserException e) {
                log.error("Exception while getting index_count ", e);
                return null;
            }
        }
    }

    public class UrlTreesUrlsProvider extends ValueProvider {
        @Override
        public Long provideValue(final HostDbHostInfo hostInfo, final Long nodeId, final YandexSearchShard searchShard) {
            try {
                return tblUrlTreesDao.getUrls(hostInfo, nodeId);
            } catch (InternalException e) {
                log.error("Exception while getting index_count ", e);
                return null;
            }
        }
    }

    public class ExternalLinksCountProvider extends ValueProvider {
        @Override
        public Long provideValue(final HostDbHostInfo hostInfo, final Long nodeId, final YandexSearchShard searchShard) {
            try {
                return linksCacheService.checkCacheAndGetLinksCount(hostInfo, searchShard);
            } catch (InternalException e) {
                log.error("Exception while getting links_count ", e);
                return null;
            } catch (UserException e) {
                log.error("Exception while getting links_count ", e);
                return null;
            }
        }
    }

    public static class IndexHistoryMapper implements ParameterizedMapRowMapper<Date, Long> {
        private final String fieldName;

        protected IndexHistoryMapper(final String fieldName) {
            this.fieldName = fieldName;
        }

        @Override
        public Pair<Date, Long> mapRow(ResultSet resultSet, int i) throws SQLException {
            return new Pair<Date, Long>(resultSet.getDate(FIELD_TIME), resultSet.getLong(fieldName));
        }
    }

    public static interface IndexHistoryAliveHandler {
        public void handle();
    }

    private static final ParameterizedMapRowMapper<Date, Long> errorHistoryForTrendRowMapper = new IndexHistoryMapper(FIELD_COUNT);

    @Required
    public void setHostDbHostInfoService(HostDbHostInfoService hostDbHostInfoService) {
        this.hostDbHostInfoService = hostDbHostInfoService;
    }

    @Required
    public void setLinksCacheService(LinksCacheService linksCacheService) {
        this.linksCacheService = linksCacheService;
    }

    @Required
    public void setHostInfoService(HostInfoService hostInfoService) {
        this.hostInfoService = hostInfoService;
    }

    @Required
    public void setHistorySaverThreadCount(int historySaverThreadCount) {
        this.historySaverThreadCount = historySaverThreadCount;
    }

    @Required
    public void setTblUsersHostsDao(TblUsersHostsDao tblUsersHostsDao) {
        this.tblUsersHostsDao = tblUsersHostsDao;
    }

    @Required
    public void setTblPeriodicCounterDao(TblPeriodicCounterDao tblPeriodicCounterDao) {
        this.tblPeriodicCounterDao = tblPeriodicCounterDao;
    }

    @Required
    public void setTblUrlTreesDao(TblUrlTreesDao tblUrlTreesDao) {
        this.tblUrlTreesDao = tblUrlTreesDao;
    }

    @Required
    public void setUrlTreeService(UrlTreeService urlTreeService) {
        this.urlTreeService = urlTreeService;
    }
}
