package ru.yandex.webmaster3.worker.antispam.threats;

import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.Iterables;
import com.google.common.collect.Range;
import org.apache.commons.lang3.tuple.Pair;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;

import ru.yandex.webmaster3.core.WebmasterException;
import ru.yandex.webmaster3.core.checklist.data.SiteProblemContent;
import ru.yandex.webmaster3.core.data.WebmasterHostId;
import ru.yandex.webmaster3.core.http.WebmasterErrorResponse;
import ru.yandex.webmaster3.core.util.IdUtils;
import ru.yandex.webmaster3.core.util.environment.YandexEnvironmentProvider;
import ru.yandex.webmaster3.core.util.environment.YandexEnvironmentType;
import ru.yandex.webmaster3.storage.checklist.data.ProblemSignal;
import ru.yandex.webmaster3.storage.checklist.data.RealTimeSiteProblemInfo;
import ru.yandex.webmaster3.storage.checklist.service.SiteProblemsNotificationService;
import ru.yandex.webmaster3.storage.clickhouse.TableType;
import ru.yandex.webmaster3.storage.host.CommonDataState;
import ru.yandex.webmaster3.storage.settings.SettingsService;
import ru.yandex.webmaster3.storage.util.clickhouse2.AbstractClickhouseDao;
import ru.yandex.webmaster3.storage.util.clickhouse2.CHPrimitiveType;
import ru.yandex.webmaster3.storage.util.clickhouse2.CHTable;
import ru.yandex.webmaster3.storage.util.yt.AsyncTableReader;
import ru.yandex.webmaster3.storage.util.yt.YtCypressService;
import ru.yandex.webmaster3.storage.util.yt.YtPath;
import ru.yandex.webmaster3.storage.util.yt.YtTableReadDriver;
import ru.yandex.webmaster3.storage.yql.YqlQueryBuilder;
import ru.yandex.webmaster3.storage.ytimport.YtClickhouseDataLoad;
import ru.yandex.webmaster3.storage.ytimport.YtClickhouseDataLoadType;
import ru.yandex.webmaster3.worker.TaskSchedule;
import ru.yandex.webmaster3.worker.turbo.AbstractYqlPrepareImportTask;

import static ru.yandex.webmaster3.storage.host.CommonDataType.LAST_IMPORTED_OWNER_THREAT_CHANGES;

/**
 * Created by Oleg Bazdyrev on 26/12/2019.
 */
public class ImportOwnerThreatsTask extends AbstractYqlPrepareImportTask {
    private static final int BATCH_SIZE = 2000;
    private static final int LINES_COUNT = 256;
    private static final Pattern TABLE_PATTERN = Pattern.compile("[0-9]+");
    private static final Duration PROCESSED_MAX_DATA_AGE = Duration.standardDays(1L);
    private static final CHTable OWNER_THREATS_TABLE = CHTable.builder()
            .database(AbstractClickhouseDao.DB_WEBMASTER3_CHECKLIST)
            .name("owner_threats_%s")
            .sharded(false)
            .partitionBy("toYYYYMM(date)")
            .keyField("date", CHPrimitiveType.Date)
            .keyField("owner", CHPrimitiveType.String)
            .keyField("host", CHPrimitiveType.String)
            .field("actual_since", CHPrimitiveType.UInt64)
            .field("threats", CHPrimitiveType.String)
            .build();

    @Autowired
    private SettingsService settingsService;
    @Autowired
    private SiteProblemsNotificationService siteProblemsNotificationService;


    @Value("${external.yt.service.arnold}://home/webmaster/prod/antiall/changes")
    private YtPath changesDir;
    @Value("${external.yt.service.arnold}://home/webmaster/prod/antiall/notifications")
    private YtPath notificationsDir;

    @Override
    protected CHTable getTable() {
        return OWNER_THREATS_TABLE;
    }

    @Override
    protected TableType getTableType() {
        return TableType.OWNER_THREATS;
    }

    @Override
    protected int getShardsCount() {
        return 1;
    }

    @Override
    protected YqlQueryBuilder prepareIntermediateTable(YtClickhouseDataLoad imprt) {
        int shardCount = getShardsCount();

        YqlQueryBuilder yqlQueryBuilder = YqlQueryBuilder.newBuilder()
                .cluster(tablePath)
                .appendText("PRAGMA yt.MaxRowWeight = '128M';\n")
                .appendText("INSERT INTO " + INTERMEDIATE_TABLE)
                .appendText("SELECT ShardId, RowId, Compress::Gzip(String::JoinFromList(AGGREGATE_LIST(data), ''), 6)" +
                        " as data FROM (\n")
                .appendText("SELECT")
                .appendText("(Digest::Fnv64(Owner) % " + shardCount + ") as ShardId,")
                .appendText("((Digest::Fnv64(Owner) / " + shardCount + ") % " + LINES_COUNT + ") as RowId,")
                .appendText("(cast(CurrentUtcDate() as String) || '\t' || " +
                        "Owner || '\t' || " +
                        "Host || '\t' || " +
                        "cast(ActualSince as String) || '\\t' || " +
                        "String::Base64Encode(Threats) || '\n')").appendText("as data ")
                .appendText("FROM")
                .appendTable(imprt.getSourceTable()).appendText(" WHERE Owner <> ''")
                .appendText(") \n GROUP BY ShardId, RowId;")
                .appendText("COMMIT;\n\n");

        return yqlQueryBuilder;
    }

    @Override
    protected YtClickhouseDataLoad rename(YtClickhouseDataLoad imprt) throws Exception {
        // импорт изменений и рассылка оповещений
        // у нас есть защита от повторных отправок, так что не будем заморачиваться с ретраями
        ytService.withoutTransaction(cypressService -> {
            long lastProcessedTimestamp =
                    Optional.ofNullable(settingsService.getSettingUncached(LAST_IMPORTED_OWNER_THREAT_CHANGES))
                            .map(CommonDataState::getValue).map(Long::valueOf).orElse(0L);
            List<YtPath> tablesToProcess =
                    cypressService.list(changesDir).stream().filter(table -> TABLE_PATTERN.matcher(table.getName()).matches())
                            .sorted().collect(Collectors.toList());
            log.info("Found {} changes tables to process: {}", tablesToProcess.size(), tablesToProcess);
            for (YtPath table : tablesToProcess) {
                long tableTimestamp = Long.parseLong(table.getName());
                if (tableTimestamp <= lastProcessedTimestamp) {
                    continue;
                }
                // process notifications
                processNotifications(cypressService, YtPath.path(notificationsDir, String.valueOf(tableTimestamp)));

                lastProcessedTimestamp = tableTimestamp;
                settingsService.update(LAST_IMPORTED_OWNER_THREAT_CHANGES, String.valueOf(lastProcessedTimestamp));
            }

            if (YandexEnvironmentProvider.getEnvironmentType() == YandexEnvironmentType.PRODUCTION) {
                cleanup(cypressService);
            }

            return true;
        });

        return super.rename(imprt);
    }

    private void processNotifications(YtCypressService cypressService, YtPath table) throws InterruptedException {
        log.info("Processing changes table {}", table);
        Map<WebmasterHostId, Pair<RealTimeSiteProblemInfo,ProblemSignal>> map = new HashMap<>();
        AsyncTableReader<HostWithThreats> tableReader = new AsyncTableReader<>(cypressService, table, Range.all(),
                YtTableReadDriver.createYSONDriver(HostWithThreats.class)).withThreadName("threats-changes-reader");
        try (var iterator = tableReader.read()) {
            while (iterator.hasNext()) {
                HostWithThreats row = iterator.next();
                if (Boolean.TRUE.equals(row.getFixed())) {
                    continue; // вроде пока нечего сообщать о пофикшенных угрозах
                }
                WebmasterHostId hostId = IdUtils.urlToHostId(row.getHost());
                // отсылаем сообщение, если надо
                ProblemSignal problemSignal = new ProblemSignal(new SiteProblemContent.Threats(), DateTime.now());
                log.info("Pretending to send threat notification for host {}", row.getHost());
                map.put(hostId,Pair.of(null,problemSignal));
                if (map.size() >= BATCH_SIZE){
                    siteProblemsNotificationService.sendNotification(map);
                    map.clear();
                }
            }
        } catch (IOException e) {
            throw new WebmasterException("Error reading table",
                    new WebmasterErrorResponse.YTServiceErrorResponse(getClass(), e), e);
        }
        if (map.size() > 0){
            siteProblemsNotificationService.sendNotification(map);
        }
    }

    private void cleanup(YtCypressService cypressService) {
        long maxTimestamp = DateTime.now().minus(PROCESSED_MAX_DATA_AGE).getMillis();
        for (YtPath table : Iterables.concat(cypressService.list(changesDir), cypressService.list(notificationsDir))) {
            long tableTimestamp = Long.parseLong(table.getName());
            if (tableTimestamp < maxTimestamp) {
                log.info("Removing old table {}", table);
                cypressService.remove(table);
            }
        }
    }

    @lombok.Value
    private static final class HostWithThreats {
        @JsonProperty("Owner")
        String owner;
        @JsonProperty("Host")
        String host;
        @JsonProperty("ActualSince")
        Long actualSince;
        @JsonProperty("Fixed")
        Boolean fixed;
        @JsonProperty("Threats")
        byte[] threats;
    }

    @Override
    protected YtClickhouseDataLoadType getImportType() {
        return YtClickhouseDataLoadType.OWNER_THREATS;
    }

    @Override
    public TaskSchedule getSchedule() {
        return TaskSchedule.startByCron("0 0/30 * * * *");
    }
}
