package ru.yandex.webmaster3.worker.hoststat;

import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;

import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Range;
import lombok.RequiredArgsConstructor;
import lombok.Setter;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import org.springframework.beans.factory.annotation.Autowired;

import ru.yandex.webmaster3.core.WebmasterException;
import ru.yandex.webmaster3.core.checklist.data.SiteProblemContent;
import ru.yandex.webmaster3.core.checklist.data.SiteProblemStorageType;
import ru.yandex.webmaster3.core.checklist.data.SiteProblemTypeEnum;
import ru.yandex.webmaster3.core.data.WebmasterHostId;
import ru.yandex.webmaster3.core.http.WebmasterErrorResponse;
import ru.yandex.webmaster3.core.util.IdUtils;
import ru.yandex.webmaster3.core.util.environment.YandexEnvironmentProvider;
import ru.yandex.webmaster3.core.util.environment.YandexEnvironmentType;
import ru.yandex.webmaster3.core.util.json.JsonMapping;
import ru.yandex.webmaster3.core.worker.task.PeriodicTaskType;
import ru.yandex.webmaster3.storage.abt.AbtService;
import ru.yandex.webmaster3.storage.abt.model.Experiment;
import ru.yandex.webmaster3.storage.checklist.data.ProblemSignal;
import ru.yandex.webmaster3.storage.checklist.service.SiteProblemsNotificationService;
import ru.yandex.webmaster3.storage.clickhouse.TableType;
import ru.yandex.webmaster3.storage.events.data.WMCEventContent;
import ru.yandex.webmaster3.storage.events.service.WMCEventsService;
import ru.yandex.webmaster3.storage.host.CommonDataState;
import ru.yandex.webmaster3.storage.hoststat.HostProblemsCHDao;
import ru.yandex.webmaster3.storage.hoststat.HostProblemsCHDao.F;
import ru.yandex.webmaster3.storage.settings.SettingsService;
import ru.yandex.webmaster3.storage.util.clickhouse2.CHField;
import ru.yandex.webmaster3.storage.util.clickhouse2.CHTable;
import ru.yandex.webmaster3.storage.util.yt.AsyncTableReader;
import ru.yandex.webmaster3.storage.util.yt.YtCypressService;
import ru.yandex.webmaster3.storage.util.yt.YtNode;
import ru.yandex.webmaster3.storage.util.yt.YtPath;
import ru.yandex.webmaster3.storage.util.yt.YtTableReadDriver;
import ru.yandex.webmaster3.storage.yql.YqlFunctions;
import ru.yandex.webmaster3.storage.yql.YqlQueryBuilder;
import ru.yandex.webmaster3.storage.ytimport.YtClickhouseDataLoad;
import ru.yandex.webmaster3.storage.ytimport.YtClickhouseDataLoadType;
import ru.yandex.webmaster3.worker.TaskSchedule;
import ru.yandex.webmaster3.worker.turbo.AbstractYqlPrepareImportTask;

import static ru.yandex.webmaster3.storage.host.CommonDataType.LAST_IMPORTED_HOST_PROBLEMS_CHANGES;

/**
 * Created by Oleg Bazdyrev on 15/03/2021.
 */
@RequiredArgsConstructor(onConstructor_ = @Autowired)
public class ImportHostProblemsTask extends AbstractYqlPrepareImportTask {

    private static final Duration CHANGES_TABLES_AGE_FOR_REMOVE = Duration.standardDays(3L);
    private static final int LINES_COUNT = 256;


    private static final Map<String, String> SOURCE_EXPRESSIONS = new ImmutableMap.Builder<String, String>()
            .put(F.HOST_ID, "$url2HostId(Host)")
            .put(F.TYPE, "Type")
            .put(F.ACTUAL_SINCE, "cast(nvl(ActualSince, 0) as String)")
            .put(F.LAST_UPDATE, "cast(nvl(LastUpdate, 0) as String)")
            .put(F.DATA, "String::ReplaceAll(Data, @@\\@@, @@\\\\@@)")
            .build();

    @Setter
    private AbtService abtService;
    @Setter
    private SettingsService settingsService;
    @Setter
    private WMCEventsService wmcEventsService;
    @Setter
    private YtPath changesDir;
    @Setter
    private SiteProblemsNotificationService siteProblemsNotificationService;

    @Override
    protected int getShardsCount() {
        return 1; // no sharding
    }

    @Override
    protected YqlQueryBuilder prepareIntermediateTable(YtClickhouseDataLoad imprt) {
        String fields = getTable().getFields().stream().map(CHField::getName).map(SOURCE_EXPRESSIONS::get)
                .collect(Collectors.joining(" || '\\t' || \n", "\n(", " || '\\n')\n"));

        YqlQueryBuilder yqlQueryBuilder = YqlQueryBuilder.newBuilder()
                .cluster(tablePath)
                .appendFDefinition(YqlFunctions.URL_2_HOST_ID)
                .appendText("PRAGMA yt.MaxRowWeight = '128M';\n")
                .appendText("INSERT INTO " + INTERMEDIATE_TABLE)
                .appendText("SELECT ShardId, RowId, Compress::Gzip(String::JoinFromList(AGGREGATE_LIST(data), ''), 6) as data FROM (\n")
                .appendText("SELECT (Digest::Fnv64($url2HostId(Host)) % " + getShardsCount() + ") as ShardId,")
                .appendText("((Digest::Fnv64($url2HostId(Host)) / " + getShardsCount() + ") % " + LINES_COUNT + ") as RowId,")
                .appendText(fields).appendText("as data ")
                .appendText("FROM")
                .appendTable(imprt.getSourceTable())
                .appendText(") \n GROUP BY ShardId, RowId;")
                .appendText("COMMIT;\n\n");

        return yqlQueryBuilder;
    }

    @Override
    // TODO многовато копипасты
    protected YtClickhouseDataLoad rename(YtClickhouseDataLoad imprt) throws Exception {
        // включаем таблицу, чтобы прорастала.
        imprt = super.rename(imprt);
        long imprtTimestamp = Long.parseLong(imprt.getData());
        // проходимся по изменениям, рассылаем сообщения
        ytService.withoutTransaction(cypressService -> {
            long lastProcessedTimestamp = Optional.ofNullable(settingsService.getSettingUncached(LAST_IMPORTED_HOST_PROBLEMS_CHANGES))
                    .map(CommonDataState::getValue).map(Long::valueOf).orElse(0L);
            List<YtPath> tablesToProcess = cypressService.list(changesDir);
            tablesToProcess.sort(null);
            log.info("Found {} changes tables to process: {}", tablesToProcess.size(), tablesToProcess);
            for (YtPath table : tablesToProcess) {
                long tableTimestamp = Long.parseLong(table.getName());
                if (tableTimestamp > imprtTimestamp) {
                    break;
                }
                if (tableTimestamp <= lastProcessedTimestamp) {
                    continue;
                }
                // process notifications
                processChanges(cypressService, table);

                lastProcessedTimestamp = tableTimestamp;
                settingsService.update(LAST_IMPORTED_HOST_PROBLEMS_CHANGES, String.valueOf(lastProcessedTimestamp));
            }

            if (YandexEnvironmentProvider.getEnvironmentType() == YandexEnvironmentType.PRODUCTION) {
                cleanup(cypressService);
            }
            return true;
        });
        return imprt;
    }

    private void processChanges(YtCypressService cypressService, YtPath table) throws InterruptedException {
        log.info("Processing changes table {}", table);
        // check params
        YtNode node = cypressService.getNode(table);
        if (!node.getNodeMeta().has("force_acceptance")) {
            long rows = node.getNodeMeta().get("row_count").asLong();
            Preconditions.checkState(rows < 150000, "Too many changes! Interrupting");
        }

        AsyncTableReader<ChangesRecord> tableReader = new AsyncTableReader<>(cypressService, table, Range.all(),
                YtTableReadDriver.createYSONDriver(ChangesRecord.class)).withThreadName("host-problems-changes-reader");
        try (var iterator = tableReader.read()) {
            while (iterator.hasNext()) {
                ChangesRecord record = iterator.next();
                WebmasterHostId hostId = IdUtils.urlToHostId(record.getHost());
                SiteProblemContent content = JsonMapping.readValue(record.getData(), SiteProblemContent.class);
                ProblemSignal problemSignal = new ProblemSignal(content, new DateTime(record.getLastUpdate()));
                DateTime actualSince = record.actualSince == null ? null : new DateTime(record.actualSince);
                WMCEventContent event = siteProblemsNotificationService.createUserMessageEvent(hostId, problemSignal, null, actualSince);
                SiteProblemTypeEnum problemType = content.getProblemType();
                if (event != null) {
                    log.info("About to send event {} of problem {} to host {}", event.getType(), problemType, hostId);
                    if (problemType.getStorageType() == SiteProblemStorageType.HOST_PROBLEM) {
                        // отправляем только "наши" алерты
                        wmcEventsService.addEvent(event);
                    }
                }
            }
        } catch (IOException e) {
            throw new WebmasterException("Error reading table", new WebmasterErrorResponse.YTServiceErrorResponse(getClass(), e), e);
        }
    }

    private void cleanup(YtCypressService cypressService) {
        long maxTimestamp = DateTime.now().minus(CHANGES_TABLES_AGE_FOR_REMOVE).getMillis();
        for (YtPath table : cypressService.list(changesDir)) {
            long tableTimestamp = Long.parseLong(table.getName());
            if (tableTimestamp < maxTimestamp) {
                log.info("Removing old table {}", table);
                cypressService.remove(table);
            }
        }
    }

    @Override
    protected CHTable getTable() {
        return HostProblemsCHDao.TABLE;
    }

    @Override
    protected TableType getTableType() {
        return TableType.HOST_PROBLEMS;
    }

    @Override
    protected YtClickhouseDataLoadType getImportType() {
        return YtClickhouseDataLoadType.HOST_PROBLEMS;
    }

    @Override
    public PeriodicTaskType getType() {
        return PeriodicTaskType.IMPORT_HOST_PROBLEMS;
    }

    @Override
    public TaskSchedule getSchedule() {
        return TaskSchedule.startByCron("30 29 * * * *");
    }

    @lombok.Value
    private static class ChangesRecord {
        @JsonProperty("Host")
        String host;
        @JsonProperty("Type")
        String type;
        @JsonProperty("ActualSince")
        Long actualSince;
        @JsonProperty("LastUpdate")
        Long lastUpdate;
        @JsonProperty("Data")
        String data;
    }

}
