package ru.yandex.webmaster3.storage.searchurl;

import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.tuple.Pair;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import ru.yandex.webmaster3.core.data.WebmasterHostId;
import ru.yandex.webmaster3.core.http.UserContext;
import ru.yandex.webmaster3.core.metrika.counters.MetrikaCountersUtil;
import ru.yandex.webmaster3.core.searchbase.SearchBaseDates;
import ru.yandex.webmaster3.core.searchbase.SearchBaseUpdateInfo;
import ru.yandex.webmaster3.core.sitestructure.RawSearchUrlStatusEnum;
import ru.yandex.webmaster3.core.sitestructure.SearchUrlStatusEnum;
import ru.yandex.webmaster3.core.sitestructure.SearchUrlStatusUtil;
import ru.yandex.webmaster3.core.util.IdUtils;
import ru.yandex.webmaster3.core.util.TimeUtils;
import ru.yandex.webmaster3.storage.abt.AbtService;
import ru.yandex.webmaster3.storage.abt.model.Experiment;
import ru.yandex.webmaster3.storage.clickhouse.table.SearchUrlsTable;
import ru.yandex.webmaster3.storage.metrika.MetrikaCrawlStateService;
import ru.yandex.webmaster3.storage.searchbase.SearchBaseUpdatesService;
import ru.yandex.webmaster3.storage.searchurl.offline.SearchBaseImportTablesService;
import ru.yandex.webmaster3.storage.searchurl.offline.dao.SearchBaseImportedTable;
import ru.yandex.webmaster3.storage.searchurl.offline.data.SearchBaseImportTaskType;
import ru.yandex.webmaster3.storage.searchurl.samples.dao.SearchUrlEventConditions;
import ru.yandex.webmaster3.storage.searchurl.samples.dao.SearchUrlEventSamplesCHDao;
import ru.yandex.webmaster3.storage.searchurl.samples.dao.SearchUrlSamplesCHDao;
import ru.yandex.webmaster3.storage.searchurl.samples.data.*;
import ru.yandex.webmaster3.storage.turbo.service.TurboSearchUrlsStatisticsService;
import ru.yandex.webmaster3.storage.util.clickhouse2.condition.Condition;
import ru.yandex.webmaster3.storage.util.clickhouse2.condition.Operator;
import ru.yandex.webmaster3.storage.util.clickhouse2.condition.TimestampCondition;

import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;

/**
 * @author avhaliullin
 */
@Service
@RequiredArgsConstructor(onConstructor_ = @Autowired)
public class SearchUrlSamplesService {
    private static final Logger log = LoggerFactory.getLogger(SearchUrlSamplesService.class);
    public static final int NUMBER_OF_FRESH_SAMPLES_IN_LIST = 5000;

    private final AbtService abtService;
    private final SearchUrlSamplesCHDao searchUrlSamplesCHDao;
    private final SearchUrlEventSamplesCHDao searchUrlEventSamplesCHDao;
    private final SearchBaseUpdatesService searchBaseUpdatesService;
    private final SearchBaseImportTablesService searchBaseImportTablesService;
    private final TurboSearchUrlsStatisticsService turboSearchUrlsStatisticsService;
    private final SearchUrlFreshSamplesService searchUrlFreshSamplesService;
    private final MetrikaCrawlStateService metrikaCrawlStateService;

    public long getSearchUrlSamplesCount(WebmasterHostId hostId, Condition filters, Condition freshFilter) {
        SearchBaseImportedTable table = searchBaseImportTablesService.getTable(SearchBaseImportTaskType.SEARCH_URL_SAMPLES);
        SearchUrlsTable shardTable = new SearchUrlsTable(table);
        return searchUrlSamplesCHDao.getSamplesCount(shardTable, hostId, filters) + searchUrlFreshSamplesService.getSearchUrlSamplesCount(hostId, freshFilter);
    }

    private Pair<Long, Long> findShiftForFreshAndBase(WebmasterHostId hostId, Condition filters, Condition freshFilter, long limitFrom, SearchUrlsTable shardTable) {
        final List<Pair<String, Long>> mapCount = searchUrlFreshSamplesService.countGroupingByBaseDate(hostId, freshFilter);
        final List<Pair<String, Long>> pairs = searchUrlEventSamplesCHDao.countGropingByBaseDate(shardTable, hostId, filters);
        return searchUrlFreshSamplesService.findSkipForFreshAndBase(mapCount, pairs, limitFrom);
    }

    public List<SearchUrlSample> getSearchUrlSamples(WebmasterHostId hostId, Condition filters, Condition freshFilter, long limitFrom, long limitSize) {
        SearchBaseImportedTable table = searchBaseImportTablesService.getTable(SearchBaseImportTaskType.SEARCH_URL_SAMPLES);
        SearchUrlsTable shardTable = new SearchUrlsTable(table);
        final TurboSearchUrlsStatisticsService.TurboSourceStatuses turboSourceStatuses = turboSearchUrlsStatisticsService.getTurboSourceStatuses(hostId);
        List<SearchUrlSample> searchUrlSamples = new ArrayList<>();
        long skip = 0;
        long size = limitSize;
        if (abtService.isInExperiment(UserContext.currentUserId(), Experiment.SEARCH_URL_FRESH)) {
            skip = limitFrom;
            long endPos = skip + size;
            if (endPos <= NUMBER_OF_FRESH_SAMPLES_IN_LIST) {
                searchUrlSamples.addAll(searchUrlFreshSamplesService.getSearchUrlSamples(hostId, freshFilter, 0, endPos));
                limitFrom = 0;
                limitSize = endPos;
            } else {
                final List<Pair<String, Long>> mapCount = searchUrlFreshSamplesService.countGroupingByLastAccess(hostId, freshFilter);
                final List<Pair<String, Long>> pairs = searchUrlSamplesCHDao.countGroupingByLastAccess(shardTable, hostId, filters);
                final Pair<Long, Long> calculate = searchUrlFreshSamplesService.findSkipForFreshAndBase(mapCount, pairs, limitFrom);
                limitFrom = calculate.getRight();
                searchUrlSamples.addAll(searchUrlFreshSamplesService.getSearchUrlSamples(hostId, freshFilter, calculate.getLeft(), limitSize));
                skip = 0;
            }
        }

        String domain = MetrikaCountersUtil.hostToPunycodeDomain(hostId);
        boolean isMetrikaCrawlEnabled = metrikaCrawlStateService.getDomainCrawlStateCached(domain).hasEnabled();
        searchUrlSamples.addAll(searchUrlSamplesCHDao.getSamples(shardTable, hostId, filters, limitFrom, limitSize)
                .stream()
                .map(sample -> new SearchUrlSample(
                        sample.getUrl(),
                        IdUtils.hostIdToUrl(hostId) + sample.getUrl(),
                        sample.getTitle(),
                        sample.getLastAccess(),
                        remapStatusInfo(turboSourceStatuses, sample.getStatusInfo())))
                .collect(Collectors.toList()));

        return searchUrlSamples
                .stream()
                .filter(e -> !(e  == null || e.getLastAccess() == null))
                .sorted(Comparator.comparing(SearchUrlSample::getLastAccess).reversed())
                .skip(skip)
                .limit(size)
                .collect(Collectors.toList());
    }

    public Optional<SearchUrlSample> getSearchUrlSample(WebmasterHostId hostId, String path) {
        SearchBaseImportedTable table = searchBaseImportTablesService.getTable(SearchBaseImportTaskType.SEARCH_URL_SAMPLES);
        SearchUrlsTable shardTable = new SearchUrlsTable(table);
        final TurboSearchUrlsStatisticsService.TurboSourceStatuses turboSourceStatuses = turboSearchUrlsStatisticsService.getTurboSourceStatuses(hostId);
        String domain = MetrikaCountersUtil.hostToPunycodeDomain(hostId);
        boolean isMetrikaCrawlEnabled = metrikaCrawlStateService.getDomainCrawlStateCached(domain).hasEnabled();

        return searchUrlFreshSamplesService.
                getSearchUrlSample(hostId, path)
                .or(() ->
                        searchUrlSamplesCHDao.getSample(shardTable, hostId, path)
                                .map(sample -> new SearchUrlSample(
                                        sample.getUrl(),
                                        IdUtils.hostIdToUrl(hostId) + sample.getUrl(),
                                        sample.getTitle(),
                                        sample.getLastAccess(),
                                        remapStatusInfo(turboSourceStatuses, sample.getStatusInfo()))));
    }

    public String getUrlSamplesDataVersion(WebmasterHostId hostId) {
        return searchBaseImportTablesService.getTable(SearchBaseImportTaskType.SEARCH_URL_SAMPLES).getFullTableName();
    }

    public long getSearchUrlEventSamplesCount(WebmasterHostId hostId, Condition filters, Condition freshFilter) {
        SearchBaseImportedTable table = searchBaseImportTablesService.getTable(SearchBaseImportTaskType.SEARCH_URL_EVENT_SAMPLES);
        SearchUrlsTable shardTable = new SearchUrlsTable(table);
        return searchUrlEventSamplesCHDao.getSamplesCount(shardTable, hostId, filters) +
                searchUrlFreshSamplesService.getSearchUrlSamplesCount(hostId, freshFilter);
    }

    public long getValidFromMetrikaNewEventSamplesCount(WebmasterHostId hostId) {
        var baseUpdates = searchBaseUpdatesService.getSearchBaseUpdates();
        var baseSwitchDate = baseUpdates.getCurrentBase().getBaseCollectionDate().toDateTime();

        Condition condition = Condition.trueCondition();
        condition = condition.andThen(new TimestampCondition(SearchUrlEventSamplesCHDao.F.EX_VALID_FROM_METRIKA_LAST_ACCESS.getName(),
                Operator.GREATER_THAN, 0));
        condition = condition.andThen(SearchUrlEventConditions.eventTypeEquals(SearchUrlEventType.NEW));
        condition = condition.andThen(new TimestampCondition(SearchUrlEventSamplesCHDao.F.SEARCH_BASE_DATE.getName(),
                Operator.EQUAL, baseSwitchDate));

        return getSearchUrlEventSamplesCount(hostId, condition, Condition.falseCondition());
    }

    public List<SearchUrlEventSample> getSearchUrlEventSamples(WebmasterHostId hostId, Condition filters, Condition freshFilter, long limitFrom, long limitSize) {
        List<SearchUrlEventSample> result = new ArrayList<>();
        SearchBaseImportedTable table = searchBaseImportTablesService.getTable(SearchBaseImportTaskType.SEARCH_URL_EVENT_SAMPLES);
        SearchUrlsTable shardTable = new SearchUrlsTable(table);

        long skip = 0;
        long size = limitSize;

        if (abtService.isInExperiment(UserContext.currentUserId(), Experiment.SEARCH_URL_FRESH)) {
            skip = limitFrom;
            long endPos = skip + size;
            if (endPos <= NUMBER_OF_FRESH_SAMPLES_IN_LIST) {
                result.addAll(searchUrlFreshSamplesService.getSearchUrlEventSamples(hostId, freshFilter, 0, endPos));
                limitFrom = 0;
                limitSize = endPos;
            } else {
                final Pair<Long, Long> calculate = findShiftForFreshAndBase(hostId, filters, freshFilter, limitFrom, shardTable);
                limitFrom = calculate.getRight();
                result.addAll(searchUrlFreshSamplesService.getSearchUrlEventSamples(hostId, freshFilter, calculate.getLeft(), limitSize));
                skip = 0;
            }
        }

        List<RawSearchUrlEventSample> rawSamples = searchUrlEventSamplesCHDao.getSamples(shardTable, hostId, filters, limitFrom, limitSize);
        SearchBaseDates searchBaseDates = searchBaseUpdatesService.getSearchBaseUpdates();

        final TurboSearchUrlsStatisticsService.TurboSourceStatuses turboSourceStatuses = turboSearchUrlsStatisticsService.getTurboSourceStatuses(hostId);
        String domain = MetrikaCountersUtil.hostToPunycodeDomain(hostId);
        boolean isMetrikaCrawlEnabled = metrikaCrawlStateService.getDomainCrawlStateCached(domain).hasEnabled();

        for (RawSearchUrlEventSample sample : rawSamples) {
            SearchBaseUpdateInfo baseUpdateInfo = searchBaseDates.getCollectionDate2Info().get(sample.getSearchBaseCollectDate().toInstant());
            if (baseUpdateInfo == null) {
                log.error("No info about base collected at {}", sample.getSearchBaseCollectDate());
            } else {
                result.add(new SearchUrlEventSample(
                        sample.getUrl(),
                        IdUtils.hostIdToUrl(hostId) + sample.getUrl(),
                        hideDownloadEvidence(sample.getStatusInfo(), sample.getTitle()),
                        sample.getLastAccess() == null ? null : sample.getLastAccess().toDateTime(TimeUtils.EUROPE_MOSCOW_ZONE).toLocalDate(),
                        baseUpdateInfo.getBaseSwitchDate().toDateTime(TimeUtils.EUROPE_MOSCOW_ZONE).toLocalDateTime(),
                        sample.getEventType(),
                        remapStatusInfo(turboSourceStatuses, sample.getStatusInfo())
                ));
            }
        }

        return result
                .stream()
                .filter(Objects::nonNull)
                .sorted(Comparator.comparing(SearchUrlEventSample::getSearchBaseDate).reversed())
                .skip(skip)
                .limit(size)
                .collect(Collectors.toList());
    }

    public long getExcludedUrlSamplesCount(WebmasterHostId hostId, Condition filters, Condition freshFilter) {
        SearchBaseImportedTable table = searchBaseImportTablesService.getTable(SearchBaseImportTaskType.EXCLUDED_URL_SAMPLES);
        SearchUrlsTable shardTable = new SearchUrlsTable(table);
        return searchUrlSamplesCHDao.getSamplesCount(shardTable, hostId, filters) + searchUrlFreshSamplesService.getExcludedUrlSamplesCount(hostId, freshFilter);
    }

    public List<SearchUrlSample> getExcludedUrlSamples(WebmasterHostId hostId, Condition filters, Condition freshFilter, long limitFrom, long limitSize) {
        List<SearchUrlSample> result = new ArrayList<>();
        //TODO в текущий момент данный тип не поддерживаться.
        //result.addAll(searchUrlFreshSamplesService.getExcludedUrlSamples(hostId, freshFilter, limitFrom, limitSize));

        SearchBaseImportedTable table = searchBaseImportTablesService.getTable(SearchBaseImportTaskType.EXCLUDED_URL_SAMPLES);
        SearchUrlsTable shardTable = new SearchUrlsTable(table);
        List<RawSearchUrlSample> rawSamples = searchUrlSamplesCHDao.getSamples(shardTable, hostId, filters, limitFrom, limitSize);
        final TurboSearchUrlsStatisticsService.TurboSourceStatuses turboSourceStatuses = turboSearchUrlsStatisticsService.getTurboSourceStatuses(hostId);
        String domain = MetrikaCountersUtil.hostToPunycodeDomain(hostId);
        boolean isMetrikaCrawlEnabled = metrikaCrawlStateService.getDomainCrawlStateCached(domain).hasEnabled();

        result.addAll(rawSamples.stream().map(raw -> new SearchUrlSample(
                raw.getUrl(),
                IdUtils.hostIdToUrl(hostId) + raw.getUrl(),
                hideDownloadEvidence(raw.getStatusInfo(), raw.getTitle()),
                raw.getLastAccess(),
                remapStatusInfo(turboSourceStatuses, raw.getStatusInfo()))).collect(Collectors.toList())
        );
        return result
                .stream()
                .filter(e -> !(e  == null || e.getLastAccess() == null))
                .sorted(Comparator.comparing(SearchUrlSample::getLastAccess).reversed())
                .limit(limitSize)
                .collect(Collectors.toList());

    }

    public Optional<SearchUrlSample> getExcludedUrlSample(WebmasterHostId hostId, String path) {
        SearchBaseImportedTable table = searchBaseImportTablesService.getTable(SearchBaseImportTaskType.EXCLUDED_URL_SAMPLES);
        SearchUrlsTable shardTable = new SearchUrlsTable(table);
        Optional<RawSearchUrlSample> rawSample = searchUrlSamplesCHDao.getSample(shardTable, hostId, path);
        final TurboSearchUrlsStatisticsService.TurboSourceStatuses turboSourceStatuses = turboSearchUrlsStatisticsService.getTurboSourceStatuses(hostId);
        String domain = MetrikaCountersUtil.hostToPunycodeDomain(hostId);
        boolean isMetrikaCrawlEnabled = metrikaCrawlStateService.getDomainCrawlStateCached(domain).hasEnabled();

        return searchUrlFreshSamplesService.getExcludedUrlSample(hostId, path).or(() ->
                rawSample.map(raw -> new SearchUrlSample(
                        raw.getUrl(),
                        IdUtils.hostIdToUrl(hostId) + raw.getUrl(),
                        hideDownloadEvidence(raw.getStatusInfo(), raw.getTitle()),
                        raw.getLastAccess(),
                        remapStatusInfo(turboSourceStatuses, raw.getStatusInfo()))));
    }

    public String getExcludedSamplesDataVersion(WebmasterHostId hostId) {
        return searchBaseImportTablesService.getTable(SearchBaseImportTaskType.EXCLUDED_URL_SAMPLES).getFullTableName();
    }

    public List<SearchUrlExtendedStatus> getExcludedUrlExtendedStatuses(WebmasterHostId hostId, Condition filters) {
        SearchBaseImportedTable table = searchBaseImportTablesService.getTable(SearchBaseImportTaskType.EXCLUDED_URL_SAMPLES);
        SearchUrlsTable shardTable = new SearchUrlsTable(table);
        List<Pair<RawSearchUrlStatusEnum, Integer>> rawStatuses = searchUrlSamplesCHDao.getStatuses(shardTable, hostId, filters);
        return rawStatuses
                .stream()
                .map(pair -> new SearchUrlExtendedStatus(
                        SearchUrlStatusUtil.raw2View(pair.getLeft(), false),// excluded url isn't searchable by definition
                        pair.getRight()
                ))
                .collect(Collectors.toList());
    }


    public List<SearchUrlEventSample> getSearchUrlEventSamplesLight(WebmasterHostId hostId, Condition filters, Condition freshFilter, long limitFrom, long limitSize) {
        List<SearchUrlEventSample> result = new ArrayList<>();
        SearchBaseImportedTable table = searchBaseImportTablesService.getTable(SearchBaseImportTaskType.SEARCH_URL_EVENT_SAMPLES);
        SearchUrlsTable shardTable = new SearchUrlsTable(table);

        long skip = 0;
        long size = limitSize;

        if (abtService.isInExperiment(UserContext.currentUserId(), Experiment.SEARCH_URL_FRESH)) {
            skip = limitFrom;
            long endPos = skip + size;
            if (endPos <= NUMBER_OF_FRESH_SAMPLES_IN_LIST) {
                result.addAll(searchUrlFreshSamplesService.getSearchUrlEventSamplesLight(hostId, freshFilter, 0, endPos));
                limitFrom = 0;
                limitSize = endPos;
            } else {
                final Pair<Long, Long> calculate = findShiftForFreshAndBase(hostId, filters, freshFilter, limitFrom, shardTable);
                limitFrom = calculate.getRight();
                result.addAll(searchUrlFreshSamplesService.getSearchUrlEventSamplesLight(hostId, freshFilter, calculate.getLeft(), limitSize));
                skip = 0;
            }
        }

        List<RawSearchUrlEventSample> rawSamples = searchUrlEventSamplesCHDao.getSamplesLight(shardTable, hostId, filters, limitFrom, limitSize);
        SearchBaseDates searchBaseDates = searchBaseUpdatesService.getSearchBaseUpdates();

        for (RawSearchUrlEventSample sample : rawSamples) {
            SearchBaseUpdateInfo baseUpdateInfo = searchBaseDates.getCollectionDate2Info().get(sample.getSearchBaseCollectDate().toInstant());
            if (baseUpdateInfo == null) {
                log.error("No info about base collected at " + sample.getSearchBaseCollectDate());
            } else {
                result.add(new SearchUrlEventSample(
                        sample.getUrl(),
                        IdUtils.hostIdToUrl(hostId) + sample.getUrl(),
                        sample.getTitle(),
                        sample.getLastAccess() == null ? null : sample.getLastAccess().toDateTime(TimeUtils.EUROPE_MOSCOW_ZONE).toLocalDate(),
                        baseUpdateInfo.getBaseSwitchDate().toDateTime(TimeUtils.EUROPE_MOSCOW_ZONE).toLocalDateTime(),
                        sample.getEventType(),
                        null
                ));
            }
        }
        return result
                .stream()
                .filter(e -> !(e  == null || e.getLastAccess() == null))
                .sorted(Comparator.comparing(SearchUrlEventSample::getSearchBaseDate).thenComparing(SearchUrlEventSample::getLastAccess).reversed())
                .limit(limitSize)
                .collect(Collectors.toList());
    }


    public String getEventSamplesDataVersion(WebmasterHostId hostId) {
        return searchBaseImportTablesService.getTable(SearchBaseImportTaskType.SEARCH_URL_EVENT_SAMPLES).getFullTableName();
    }

    @Nullable
    private static <T> T hideDownloadEvidence(RawUrlStatusInfo statusInfo, T data) {
        return hideDownloadEvidence(statusInfo.getStatus(), data);
    }

    @Nullable
    public static <T> T hideDownloadEvidence(RawSearchUrlStatusEnum status, T data) {
        boolean shouldHide = (
                status == RawSearchUrlStatusEnum.ROBOTS_HOST_ERROR ||
                        status == RawSearchUrlStatusEnum.ROBOTS_URL_ERROR
        );
        return shouldHide ? null : data;
    }

    private UrlStatusInfo remapStatusInfo(TurboSearchUrlsStatisticsService.TurboSourceStatuses turboSourceStatuses,
                                          RawUrlStatusInfo statusInfo) {
        if (statusInfo == null) {
            return new UrlStatusInfo(
                    SearchUrlStatusEnum.NOTHING_FOUND,
                    null,
                    null,
                    null,
                    null,
                    null,
                    null,
                    null,
                    null,
                    false,
                    false,
                    false,
                    false,
                    false
            );
        } else {
            long validFromMetrikaLastAccess = statusInfo.getValidFromMetrikaLastAccess();
            long validFromIndexNowLastAccess = statusInfo.getValidFromIndexNowLastAccess();

            return new UrlStatusInfo(
                    remapStatusEnum(statusInfo),
                    statusInfo.getAddTime(),
                    statusInfo.getBeautyUrl(),
                    hideDownloadEvidence(statusInfo, statusInfo.getHttpCode()),
                    statusInfo.getMainHost(),
                    statusInfo.getMainPath(),
                    statusInfo.getRedirectTarget(),
                    statusInfo.getRelCanonicalTarget(),
                    null,
                    TurboSearchUrlsStatisticsService.isEnabledTurboPage(statusInfo.getTurboSourceFlag(), turboSourceStatuses),
                    statusInfo.isFromSitemap(),
                    UrlStatusInfo.isValidFromMetrika(validFromMetrikaLastAccess, validFromIndexNowLastAccess),
                    UrlStatusInfo.isValidFromIndexNow(validFromMetrikaLastAccess, validFromIndexNowLastAccess),
                    false
            );
        }
    }

    private static SearchUrlStatusEnum remapStatusEnum(RawUrlStatusInfo status) {
        return SearchUrlStatusUtil.raw2View(status.getStatus(), status.isSearchable());
    }

}
