package ru.yandex.webmaster3.worker.turbo;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;

import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import com.google.common.collect.Range;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.tuple.Pair;
import org.joda.time.DateTime;
import org.joda.time.LocalDate;
import org.joda.time.format.ISODateTimeFormat;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import ru.yandex.webmaster3.core.checklist.data.SiteProblemContent;
import ru.yandex.webmaster3.core.checklist.data.SiteProblemState;
import ru.yandex.webmaster3.core.data.WebmasterHostId;
import ru.yandex.webmaster3.core.turbo.model.statistics.TurboClicksDayStatistics;
import ru.yandex.webmaster3.core.turbo.model.statistics.TurboTopUrlInfo;
import ru.yandex.webmaster3.core.util.IdUtils;
import ru.yandex.webmaster3.core.util.RetryUtils;
import ru.yandex.webmaster3.core.util.W3Collectors;
import ru.yandex.webmaster3.core.util.json.JsonMapping;
import ru.yandex.webmaster3.core.worker.task.PeriodicTaskState;
import ru.yandex.webmaster3.core.worker.task.PeriodicTaskType;
import ru.yandex.webmaster3.storage.checklist.dao.ChecklistPageSamplesService;
import ru.yandex.webmaster3.storage.checklist.dao.ChecklistSamplesType;
import ru.yandex.webmaster3.storage.checklist.data.ProblemSignal;
import ru.yandex.webmaster3.storage.checklist.data.RealTimeSiteProblemInfo;
import ru.yandex.webmaster3.storage.checklist.service.SiteProblemsService;
import ru.yandex.webmaster3.storage.host.CommonDataState;
import ru.yandex.webmaster3.storage.host.CommonDataType;
import ru.yandex.webmaster3.storage.settings.dao.CommonDataStateYDao;
import ru.yandex.webmaster3.storage.settings.data.AbstractCommonDataState;
import ru.yandex.webmaster3.storage.util.yt.AsyncTableReader;
import ru.yandex.webmaster3.storage.util.yt.YtCypressService;
import ru.yandex.webmaster3.storage.util.yt.YtException;
import ru.yandex.webmaster3.storage.util.yt.YtPath;
import ru.yandex.webmaster3.storage.util.yt.YtService;
import ru.yandex.webmaster3.storage.util.yt.YtTableReadDriver;
import ru.yandex.webmaster3.worker.PeriodicTask;
import ru.yandex.webmaster3.worker.TaskSchedule;

import static ru.yandex.webmaster3.core.checklist.data.SiteProblemTypeEnum.TURBO_INSUFFICIENT_CLICKS_SHARE;


/**
 * ishalaru
 * 07.02.2020
 **/
@Slf4j
@RequiredArgsConstructor(onConstructor_ = @Autowired)
@Component("importTurboClickDayStatsTask")
public class UpdateTurboInsuffucientClicksShareTask extends PeriodicTask<UpdateTurboInsuffucientClicksShareTask.TaskState> {

    private static final int SAVE_BATCH_SIZE = 500;

    private final ChecklistPageSamplesService checklistPageSamplesService;
    private final CommonDataStateYDao commonDataStateYDao;
    private final SiteProblemsService siteProblemsService;
    private final YtService ytService;
    @Value("${webmaster3.worker.import.turbo.clicks.day.stats.dir.yt.path}")
    private final YtPath path;

    @Override
    public Result run(UUID runId) throws Exception {

        setState(new TaskState());
        LocalDate lastImportedDate = Optional.ofNullable(commonDataStateYDao.getValue(CommonDataType.TURBO_CLICK_DAY_STATS_LAST_UPDATE))
                .map(AbstractCommonDataState::getValue)
                .map(e -> LocalDate.parse(e, ISODateTimeFormat.basicDate()))
                .orElse(LocalDate.parse("2018-01-01"));

        ytService.inTransaction(path).execute(cypressService -> {
            List<YtPath> tableForProcess = findTableToProcess(lastImportedDate, cypressService);
            if (tableForProcess.isEmpty()) {
                log.info("Did not find table to process. Last processed table: {}. Finishing task", lastImportedDate);
                return false;
            }
            state.countReadTable = tableForProcess.size();
            tableForProcess.sort(null);
            LocalDate lastProcessedDate = lastImportedDate;
            for (YtPath item : tableForProcess) {
                String date = item.getName().substring(item.getName().length() - 8);
                LocalDate tableDate = LocalDate.parse(date, ISODateTimeFormat.basicDate());
                if (!tableDate.minusDays(1).equals(lastProcessedDate)) {
                    log.error("Found gap in turbo click stats tables. Last processed: {}, found {}", lastProcessedDate, tableDate);
                    return false;
                }
                processTable(cypressService, item);
                commonDataStateYDao.update(new CommonDataState(CommonDataType.TURBO_CLICK_DAY_STATS_LAST_UPDATE, date, DateTime.now()));
                state.countProcessedTable++;
                state.processedTable.add(date);
                lastProcessedDate = tableDate;
            }
            return true;
        });

        return Result.SUCCESS;
    }

    private void processTable(YtCypressService cypressService, YtPath tablePath) throws InterruptedException {
        AsyncTableReader<ClickStatsRow> tableReader = new AsyncTableReader<>(
                cypressService,
                tablePath,
                Range.all(),
                YtTableReadDriver.createYSONDriver(ClickStatsRow.class, JsonMapping.OM)
        ).splitInParts(10000L)
                .withThreadName("turbo-clicks-day-stats-cacher")
                .withRetry(5);
        LocalDate localDate = LocalDate.parse(tablePath.getName().substring(tablePath.getName().length() - 8), ISODateTimeFormat.basicDate());
        try (AsyncTableReader.TableIterator<ClickStatsRow> it = tableReader.read()) {
            dataProcessing(localDate, it);
        } catch (IOException e) {
            throw new YtException("Unable to read table: " + tablePath, e);
        }
    }

    private void dataProcessing(LocalDate localDate, AsyncTableReader.TableIterator<ClickStatsRow> it) throws IOException, InterruptedException {
        List<TurboClicksDayStatistics> readyToSaveList = new ArrayList<>(SAVE_BATCH_SIZE);
        while (it.hasNext()) {
            final ClickStatsRow row = it.next();
            if (!row.isValid()) {
                continue;
            }
            Preconditions.checkArgument(row.totalClicks >= 0);
            Preconditions.checkArgument(row.turboClicks >= 0);
            Preconditions.checkArgument(row.autoparsedClicks >= 0);
            state.countProcessedRecords++;
            readyToSaveList.add(row.toClicksStatistics(localDate));
            if (readyToSaveList.size() >= SAVE_BATCH_SIZE) {
                saveList(readyToSaveList);
            }
        }
        saveList(readyToSaveList);
    }


    private void saveList(List<TurboClicksDayStatistics> list) throws InterruptedException {
        Map<WebmasterHostId, Pair<ProblemSignal, RealTimeSiteProblemInfo>> problemsMap = new HashMap<>();
        Map<WebmasterHostId, List<String>> samplesMap = new HashMap<>();
        Map<WebmasterHostId, TurboClicksDayStatistics> statsByHostId = list.stream().map(stats -> {
            try {
                return Pair.of(IdUtils.urlToHostId(stats.getDomain()), stats);
            } catch (IllegalArgumentException ignored) {
                // bad domain name
                return null;
            }
        }).filter(Objects::nonNull).collect(W3Collectors.toHashMap());
        Map<WebmasterHostId, RealTimeSiteProblemInfo> siteProblems = siteProblemsService.listSitesProblems(statsByHostId.keySet(), TURBO_INSUFFICIENT_CLICKS_SHARE);
        for (var entry : statsByHostId.entrySet()) {
            RetryUtils.execute(RetryUtils.instantRetry(3), () -> saveExamplesOfTurboPages(entry.getKey(), entry.getValue(), siteProblems, problemsMap, samplesMap));
        }

        siteProblemsService.updateRealTimeProblem(problemsMap);
        checklistPageSamplesService.saveSamples(samplesMap, ChecklistSamplesType.TURBO_INSUFFICIENT_CLICKS_SHARE);

        list.clear();
    }

    private void saveExamplesOfTurboPages(WebmasterHostId hostId, TurboClicksDayStatistics tbClicks,
                                          Map<WebmasterHostId, RealTimeSiteProblemInfo> siteProblems,
                                          Map<WebmasterHostId, Pair<ProblemSignal, RealTimeSiteProblemInfo>> problemsMap,
                                          Map<WebmasterHostId, List<String>> samplesMap) {
        double turboClicksShare = 1.0 * tbClicks.getTurboClicks() / tbClicks.getTotalClicks();
        double autoparsedClicksShare = 1.0 * tbClicks.getAutoparsedClicks() / tbClicks.getTurboClicks();
        boolean insufficient = turboClicksShare < 0.8;
        /* Про autoparsedClicksShare > 0.5 && insufficient:
         * Если доля кликов на страницы автопарсера слишком высока, будет неправильно показывать алерт -- в этом
         * случае мы сами виноваты в низком доле турбо-трафика. Но выключить алерт в таком случае -- правильно.
         * Поэтому для autoparsedClicksShare > 0.5 мы хотим обновить примеры, но не менять сттутс проблемы.
         */
        boolean problemIsOn = false;
        boolean shouldUpdateSamples = false;
        RealTimeSiteProblemInfo problem = siteProblems.get(hostId);
        if (tbClicks.getTotalClicks() < 20 || turboClicksShare < 0.1 || autoparsedClicksShare > 0.5 && insufficient) {
            if (problem != null && problem.getState() == SiteProblemState.PRESENT) {
                shouldUpdateSamples = true;
            } else {
                return;
            }
        }
        if (insufficient) {
            problemIsOn = true;
            shouldUpdateSamples = true;
        }

        ProblemSignal problemSignal;
        if (problemIsOn) {
            int turboClicksPercents = (int) Math.round(100.0 * turboClicksShare);
            problemSignal = new ProblemSignal(
                    new SiteProblemContent.TurboInsufficientClicksShare(turboClicksPercents), DateTime.now());
        } else {
            problemSignal = new ProblemSignal(TURBO_INSUFFICIENT_CLICKS_SHARE, SiteProblemState.ABSENT, DateTime.now());
        }

        problemsMap.put(hostId, Pair.of(problemSignal, problem));
        final List<String> samples = tbClicks.getTopUrlsWithoutTurbo().stream().map(TurboTopUrlInfo::getUrl).collect(Collectors.toList());
        samplesMap.put(hostId, shouldUpdateSamples ? samples : Collections.emptyList());
    }

    private List<YtPath> findTableToProcess(LocalDate tableImportedLastTime,
                                            YtCypressService cypressService) {
        return cypressService.list(path)
                .stream()
                .map(YtPath::getName)
                .filter(n -> n.matches("\\d{8}"))
                .filter(n -> LocalDate.parse(n, ISODateTimeFormat.basicDate()).isAfter(tableImportedLastTime))
                .sorted(Comparator.naturalOrder())
                .map(name -> YtPath.path(path, name))
                .collect(Collectors.toList());
    }

    private static class ClickStatsRow {
        private final String domain;
        private final long turboClicks;
        private final long totalClicks;
        private final long autoparsedClicks;
        private final List<TurboTopUrlInfo> topUrlsWithoutTurbo;

        public ClickStatsRow(
                @JsonProperty("domain") String domain,
                @JsonProperty("turbo_clicks") Long turboClicks,
                @JsonProperty("total_clicks") Long totalClicks,
                @JsonProperty("autoparsed_clicks") Long autoparsedClicks,
                @JsonProperty("top_urls_without_turbo") List<TurboTopUrlInfo> topUrlsWithoutTurbo) {
            this.domain = domain;
            this.turboClicks = turboClicks;
            this.totalClicks = totalClicks;
            this.autoparsedClicks = autoparsedClicks;
            this.topUrlsWithoutTurbo = Preconditions.checkNotNull(topUrlsWithoutTurbo);
            Preconditions.checkArgument(topUrlsWithoutTurbo.size() <= 100,
                    "Too many elements in list: " + topUrlsWithoutTurbo.size());
        }

        public boolean isValid() {
            return domain != null;
        }

        public TurboClicksDayStatistics toClicksStatistics(LocalDate date) {
            return new TurboClicksDayStatistics(domain, date, totalClicks, turboClicks, autoparsedClicks,
                    topUrlsWithoutTurbo.subList(0, Math.min(topUrlsWithoutTurbo.size(), 100)));
        }
    }


    @Override
    public PeriodicTaskType getType() {
        return PeriodicTaskType.UPDATE_TURBO_INSUFFICIENT_CLICKS_SHARE;
    }

    @Override
    public TaskSchedule getSchedule() {
        //return TaskSchedule.startByCron("0 15 * * * *");
        return TaskSchedule.never();
    }

    static class TaskState implements PeriodicTaskState {
        @Getter
        List<String> processedTable = new ArrayList<>();
        @Getter
        long countProcessedTable;
        @Getter
        long countReadTable;
        @Getter
        long countProcessedRecords;
    }
}
