package ru.yandex.webmaster3.worker.checklist;

import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Charsets;
import com.google.common.collect.Lists;
import com.google.common.collect.Range;
import lombok.AllArgsConstructor;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.mutable.MutableInt;
import org.apache.commons.lang3.mutable.MutableObject;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import ru.yandex.webmaster3.core.WebmasterException;
import ru.yandex.webmaster3.core.checklist.data.SiteProblemContent;
import ru.yandex.webmaster3.core.checklist.data.SiteProblemContent.MordaError.ExtendedStatus;
import ru.yandex.webmaster3.core.checklist.data.SiteProblemTypeEnum;
import ru.yandex.webmaster3.core.data.WebmasterHostId;
import ru.yandex.webmaster3.core.http.WebmasterErrorResponse;
import ru.yandex.webmaster3.core.util.IdUtils;
import ru.yandex.webmaster3.core.worker.client.WorkerClient;
import ru.yandex.webmaster3.core.worker.task.PeriodicTaskState;
import ru.yandex.webmaster3.core.worker.task.PeriodicTaskType;
import ru.yandex.webmaster3.core.worker.task.TaskResult;
import ru.yandex.webmaster3.storage.checklist.data.ProblemSignal;
import ru.yandex.webmaster3.storage.checklist.service.SiteProblemsService;
import ru.yandex.webmaster3.storage.host.AllHostsCacheService;
import ru.yandex.webmaster3.storage.host.CommonDataState;
import ru.yandex.webmaster3.storage.settings.dao.CommonDataStateYDao;
import ru.yandex.webmaster3.storage.settings.data.AbstractCommonDataState;
import ru.yandex.webmaster3.storage.util.yt.AsyncTableReader;
import ru.yandex.webmaster3.storage.util.yt.YtPath;
import ru.yandex.webmaster3.storage.util.yt.YtRowMapper;
import ru.yandex.webmaster3.storage.util.yt.YtService;
import ru.yandex.webmaster3.storage.util.yt.YtTableReadDriver;
import ru.yandex.webmaster3.worker.PeriodicTask;
import ru.yandex.webmaster3.worker.TaskSchedule;
import ru.yandex.wmtools.common.util.http.YandexHttpStatus;

import static ru.yandex.webmaster3.storage.host.CommonDataType.LAST_MORDA_INFO_PROCESSED;

/**
 * @author avhaliullin
 */
@Slf4j
@Component("mordaInfoUpdateTask")
@RequiredArgsConstructor(onConstructor_ = @Autowired)
public class MordaInfoUpdateTask extends PeriodicTask<PeriodicTaskState> {
    private static final int TOTAL_THREADS = 8;
    private static final int BATCH_SIZE = 2000;
    private static final int TABLE_CHUNK_SIZE = 10_000;
    private static final int LOGGING_DENSITY = 100;

    // не проверяем записи старше 10 дней (в случае. если нет последней даты выполнения таски)
    private static final Duration MAX_RECORD_AGE_FOR_REDIRECT_CHECK = Duration.standardDays(10L);

    private final AllHostsCacheService allHostsCacheService;
    private final YtService ytService;
    private final CommonDataStateYDao commonDataStateYDao;
    private final SiteProblemsService siteProblemsService;
    @Qualifier("lbWorkerClient")
    private final WorkerClient workerClient;

    @Value("${webmaster3.worker.mordaInfoUpdateTask.problem.table.path}")
    private YtPath faceProblemTable;


    @Override
    public PeriodicTaskType getType() {
        return PeriodicTaskType.MORDA_INFO_UPDATE;
    }

    @Override
    public Result run(UUID runId) throws Exception {
        TaskState ts = new TaskState();
        setState(ts);

        DateTime updateStarted = DateTime.now();
        DateTime lastProcessed = Optional.ofNullable(commonDataStateYDao.getValue(LAST_MORDA_INFO_PROCESSED))
                .map(AbstractCommonDataState::getLastUpdate).orElse(updateStarted.minus(MAX_RECORD_AGE_FOR_REDIRECT_CHECK));
        MutableObject<DateTime> maxProcessed = new MutableObject<>(lastProcessed);

        ExecutorService executorService = Executors.newFixedThreadPool(TOTAL_THREADS);
        ytService.inTransaction(faceProblemTable).execute(cypressService -> {
            AsyncTableReader<MordaProblemInfo> tableReader = new AsyncTableReader<>(
                    cypressService, faceProblemTable, Range.all(), YtTableReadDriver.createYSONDriver(MordaProblemInfo.class))
                    .splitInParts(TABLE_CHUNK_SIZE).withRetry(5);
            log.info("Processing morda problems table. Starting");
            try (var iterator = tableReader.read()) {
                int lineIndex = 0;
                Map<WebmasterHostId, ProblemSignal> problemsBatch = new HashMap<>(BATCH_SIZE);
                List<Callable<Void>> redirectBatch = new ArrayList<>(TABLE_CHUNK_SIZE);
                while (iterator.hasNext()) {
                    ts.totalBadCodeHosts.increment();
                    MordaProblemInfo errorInfo = iterator.next();
                    WebmasterHostId hostId = IdUtils.urlToHostId(errorInfo.getHost());
                    if (!allHostsCacheService.contains(hostId)) {
                        continue;
                    }

                    if (lineIndex++ % LOGGING_DENSITY == 0) {
                        log.info("Processing morda errors table, host {} line {}", hostId, lineIndex);
                    }
                    DateTime lastAccess = new DateTime(errorInfo.getLastAccess() * 1000L);
                    YandexHttpStatus httpStatus = YandexHttpStatus.parseCode(errorInfo.getHttpCode());
                    if (httpStatus == YandexHttpStatus.UNKNOWN) {
                        log.info("UNKNOWN http status for host {}", hostId);
                        continue; // непонятно, что с сайтом - игнорируем
                    }
                    if ((httpStatus.getCode() >= 300 && httpStatus.getCode() < 400) || httpStatus == YandexHttpStatus.EXT_HTTP_2004_REFRESH) {
                        // redirect
                        maxProcessed.setValue(ObjectUtils.max(maxProcessed.getValue(), lastAccess));
                        boolean shouldCheck = lastAccess.isAfter(lastProcessed);
                        if (shouldCheck) {
                            ts.totalRedirectCheckedHosts.increment();
                        }
                        redirectBatch.add(() -> updateMordaRedirectProblem(hostId, errorInfo, shouldCheck));
                        if (redirectBatch.size() >= TABLE_CHUNK_SIZE) {
                            executeBatch(executorService, redirectBatch);
                        }
                    } else {
                        // bad code
                        problemsBatch.put(hostId, new ProblemSignal(
                                new SiteProblemContent.MordaError(lastAccess, ExtendedStatus.DEFAULT, httpStatus, false),
                                lastAccess
                        ));
                        if (problemsBatch.size() >= BATCH_SIZE) {
                            siteProblemsService.updateCleanableProblems(problemsBatch, SiteProblemTypeEnum.MORDA_ERROR);
                            problemsBatch.clear();
                        }
                    }
                }
                siteProblemsService.updateCleanableProblems(problemsBatch, SiteProblemTypeEnum.MORDA_ERROR);
                executeBatch(executorService, redirectBatch);
            } catch (IOException e) {
                throw new WebmasterException("Error reading morda bad code table from YT",
                        new WebmasterErrorResponse.YTServiceErrorResponse(getClass(), e), e);
            } finally {
                executorService.shutdownNow();
            }
            log.info("Processing morda problems table. Finished");
            return true;
        });

        siteProblemsService.notifyCleanableProblemUpdateFinished(SiteProblemTypeEnum.MORDA_ERROR, updateStarted);
        siteProblemsService.notifyCleanableProblemUpdateFinished(SiteProblemTypeEnum.MORDA_REDIRECTS, updateStarted);
        //
        commonDataStateYDao.update(new CommonDataState(LAST_MORDA_INFO_PROCESSED,
                String.valueOf(maxProcessed.getValue().getMillis()), maxProcessed.getValue()));

        return new Result(TaskResult.SUCCESS);
    }

    private void executeBatch(ExecutorService executorService, List<Callable<Void>> batch) {
        if (batch.isEmpty()) {
            return;
        }

        try {
            List<Future<Void>> futures = executorService.invokeAll(batch);
            for (Future<Void> f : futures) {
                f.get();
            }
        } catch (InterruptedException | ExecutionException e) {
            throw new WebmasterException("Failed to execute batch",
                    new WebmasterErrorResponse.InternalUnknownErrorResponse(getClass(), null), e);
        }

        batch.clear();
    }


    private Void updateMordaRedirectProblem(WebmasterHostId hostId, MordaProblemInfo errorInfo, boolean shouldCheck) {
        siteProblemsService.touchCleanableProblemIfExists(hostId, SiteProblemTypeEnum.MORDA_ERROR);
        siteProblemsService.touchCleanableProblemIfExists(hostId, SiteProblemTypeEnum.MORDA_REDIRECTS);

        if (shouldCheck) {
            workerClient.enqueueTask(new FollowMordaRedirectsTaskData(hostId));
        }

        return null;
    }

    @Override
    public TaskSchedule getSchedule() {
        return TaskSchedule.startByCron("0 0 10 * * *");
    }

    @lombok.Value
    @AllArgsConstructor(onConstructor_ = @JsonCreator)
    public static class MordaProblemInfo {
        @JsonProperty("Host")
        String host;
        @JsonProperty("LastAccess")
        long lastAccess;
        @JsonProperty("HttpCode")
        int httpCode;
    }

    public static class TaskState implements PeriodicTaskState {
        MutableInt totalBadCodeHosts = new MutableInt(0);
        MutableInt totalRedirectHosts = new MutableInt(0);
        MutableInt totalRedirectCheckedHosts = new MutableInt(0);

        public int getTotalBadCodeHosts() {
            return totalBadCodeHosts.getValue();
        }

        public int getTotalRedirectHosts() {
            return totalRedirectHosts.getValue();
        }

        public int getTotalRedirectCheckedHosts() {
            return totalRedirectCheckedHosts.getValue();
        }
    }
}
