package ru.yandex.webmaster3.worker.feeds.logs;

import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

import com.google.common.collect.ImmutableMap;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.text.StrSubstitutor;
import org.apache.commons.lang3.tuple.Pair;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import org.joda.time.LocalDate;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import ru.yandex.webmaster3.core.util.environment.YandexEnvironmentProvider;
import ru.yandex.webmaster3.core.util.environment.YandexEnvironmentType;
import ru.yandex.webmaster3.core.worker.task.PeriodicTaskType;
import ru.yandex.webmaster3.storage.clickhouse.TableType;
import ru.yandex.webmaster3.storage.util.clickhouse2.CHField;
import ru.yandex.webmaster3.storage.util.clickhouse2.CHTable;
import ru.yandex.webmaster3.storage.util.clickhouse2.ClickhouseHost;
import ru.yandex.webmaster3.storage.util.clickhouse2.ClickhouseQueryContext;
import ru.yandex.webmaster3.storage.util.yt.YtPath;
import ru.yandex.webmaster3.storage.yql.YqlFunctions;
import ru.yandex.webmaster3.storage.yql.YqlQueryBuilder;
import ru.yandex.webmaster3.storage.ytimport.YtClickhouseDataLoad;
import ru.yandex.webmaster3.storage.ytimport.YtClickhouseDataLoadType;
import ru.yandex.webmaster3.worker.TaskSchedule;
import ru.yandex.webmaster3.worker.turbo.AbstractYqlPrepareImportTask;

import static ru.yandex.webmaster3.storage.feeds.logs.FeedsOffersLogsHistoryCHDao.F;
import static ru.yandex.webmaster3.storage.feeds.logs.FeedsOffersLogsHistoryCHDao.TABLE;
import static ru.yandex.webmaster3.storage.feeds.logs.FeedsOffersLogsHistoryCHDao.TABLE_NAME;
import static ru.yandex.webmaster3.storage.ytimport.YtClickhouseDataLoadState.DONE;

/**
 * Created by Oleg Bazdyrev on 16/12/2021.
 */
@Slf4j
@Service
@RequiredArgsConstructor(onConstructor_ = @Autowired)
public class ImportFeedsOffersLogsHistoryTask extends AbstractYqlPrepareImportTask {

    private static final long OLD_TABLE_AGE_MILLIS = TimeUnit.DAYS.toMillis(2L);
    private static final DateTimeFormatter DATE_FORMAT = DateTimeFormat.forPattern("yyyyMMdd");

    private static final int LINES_COUNT = 128;
    private static final Map<String, String> SOURCE_EXPRESSIONS = new ImmutableMap.Builder<String, String>()
            .put(F.HOST, "host")
            .put(F.FEED, "feed")
            .put(F.TIMESTAMP, "cast(`timestamp` as String)")
            .put(F.STATS, "nvl(cast(Yson::SerializeJson(stats) as String), '')")
            .put(F.ERROR_STATS, "nvl(cast(Yson::SerializeJson(error_stats) as String), '')")
            .put(F.ERRORS, "nvl(String::ReplaceAll(cast(Yson::SerializeJson(errors) as String), @@\\@@, @@\\\\@@), '')")
            .put(F.LAST_ACCESS, "cast(nvl(last_access, 0) as String)")
            .put(F.TYPE, "nvl(type, '')")
            .build();

    protected String getFullTable() {
        return TABLE_NAME;
    }

    @Override
    protected CHTable getTable() {
        return TABLE;
    }

    @Override
    protected TableType getTableType() {
        return null; // not needed
    }

    @Override
    protected YtClickhouseDataLoad init(YtClickhouseDataLoad imprt) throws Exception {
        // тупо возьмем самую последнюю таблицу и будем обрабатывать диапазон
        return ytService.withoutTransactionQuery(cypressService -> {
            String[] fromAndTo = imprt.getData().split("_");
            String from = fromAndTo[0];
            //String to = fromAndTo.length == 1 ? from : fromAndTo[1];
            List<YtPath> tables = cypressService.list(tablePath).stream().filter(table -> table.getName().compareTo(from) > 0)
                    .sorted().collect(Collectors.toList());
            if (tables.isEmpty()) {
                log.info("Expected table not found, nothing to import");
                return imprt.withState(DONE);
            }

            return imprt.withSourceTable(tables.get(0), LocalDate.now(), LocalDate.now())
                    .withData(tables.get(0).getName() + "_" + tables.get(tables.size() - 1).getName());
        });
    }

    @Override
    protected YqlQueryBuilder prepareIntermediateTable(YtClickhouseDataLoad imprt) {
        String fields = getTable().getFields().stream().map(CHField::getName).map(SOURCE_EXPRESSIONS::get)
                .collect(Collectors.joining(" || '\\t' || ", "(", " || '\\n')"));

        String[] fromAndTo = imprt.getData().split("_");
        String from = fromAndTo[0];
        String to = fromAndTo[1];

        YqlQueryBuilder yqlQueryBuilder = YqlQueryBuilder.newBuilder()
                .cluster(tablePath)
                .appendFDefinition(YqlFunctions.URL_2_HOST_ID)
                .appendText("PRAGMA yt.DefaultMemoryLimit = '4G';\n")
                .appendText("PRAGMA yt.MaxRowWeight = '128M';\n")
                .appendText("INSERT INTO " + INTERMEDIATE_TABLE)
                .appendText("SELECT ShardId, RowId, Compress::Gzip(String::JoinFromList(AGGREGATE_LIST(data), ''), 6) as data FROM (\n")
                .appendText("SELECT (Digest::Fnv64(host) % " + getShardsCount() + ") as ShardId,")
                .appendText("((Digest::Fnv64(host || feed || cast(`timestamp` as String))) % " + LINES_COUNT + ") as RowId,")
                .appendText(fields).appendText("as data ")
                .appendText("FROM RANGE(").appendTable(tablePath).appendText(", '" + from + "', '" + to + "')");
        yqlQueryBuilder
                .appendText(") \n GROUP BY ShardId, RowId;")
                .appendText("COMMIT;\n\n");

        return yqlQueryBuilder;
    }

    @Override
    protected YtClickhouseDataLoad replicate(YtClickhouseDataLoad imprt) throws Exception {
        return imprt.withNextState();
    }

    @Override
    protected YtClickhouseDataLoad rename(YtClickhouseDataLoad imprt) throws Exception {
        String database = getTable().getDatabase();
        String tempTableName = getTable().replicatedMergeTreeTableName(-1, imprt.getData().replace("-", "_"));
        // удаляем из общей таблицы подливаемую дату и вливаем свежие данные
        for (ClickhouseHost host : clickhouseServer.getHosts()) {
            boolean hasTable = !clickhouseSystemTablesCHDao.getTables(host, database, Collections.singleton(tempTableName)).isEmpty();
            if (!hasTable) {
                continue;
            }
            // получим минимальный и максимальный ts в новой таблице
            ClickhouseQueryContext.Builder ctx = ClickhouseQueryContext.useDefaults().setHost(host).setTimeout(Duration.standardMinutes(10L));
            Pair<Long, Long> minMax = clickhouseServer.queryOne(ctx, "SELECT min(timestamp), max(timestamp) FROM " + database + "." + tempTableName,
                    chRow -> Pair.of(chRow.getLong(0), chRow.getLong(1))).orElseThrow();
            StrSubstitutor substitutor = new StrSubstitutor(Map.of(
                    "DB", database,
                    "FULL_TABLE", getFullTable(),
                    "TEMP_TABLE", tempTableName,
                    "MIN_TS", minMax.getLeft(),
                    "MAX_TS", minMax.getRight(),
                    "ATTACH_PARTITIONS", IntStream.range(0, 16).mapToObj(prt -> "ATTACH PARTITION '" + prt + "' FROM " + database + "." + tempTableName)
                            .collect(Collectors.joining(","))
            ));
            for (String query : substitutor.replace(MERGE_TABLE_QUERY).split("\n\n")) {
                clickhouseServer.execute(ctx, query);
            }
        }
        return imprt.withNextState();
    }

    @Override
    protected YtClickhouseDataLoad clean(YtClickhouseDataLoad imprt) throws Exception {
        // clear source tables
        if (YandexEnvironmentProvider.getEnvironmentType() == YandexEnvironmentType.PRODUCTION) {
            long maxTableForRemoval = Long.parseLong(imprt.getSourceTable().getName()) - OLD_TABLE_AGE_MILLIS;
            ytService.withoutTransaction(cypressService -> {
                cypressService.list(tablePath).stream().filter(table -> Long.parseLong(table.getName()) < maxTableForRemoval)
                        .forEach(cypressService::remove);
                return true;
            });
        }
        return super.clean(imprt);
    }

    @Override
    protected YtClickhouseDataLoad createDistributedTables(YtClickhouseDataLoad imprt) throws Exception {
        return imprt.withNextState();
    }

    @Override
    public PeriodicTaskType getType() {
        return PeriodicTaskType.IMPORT_FEEDS_OFFERS_LOGS_HISTORY;
    }

    @Override
    protected YtClickhouseDataLoadType getImportType() {
        return YtClickhouseDataLoadType.FEEDS_OFFERS_LOGS_HISTORY;
    }

    @Override
    public TaskSchedule getSchedule() {
        return TaskSchedule.startByCron("0 12/20 * * * *");
    }

    private static final String MERGE_TABLE_QUERY = """
            ALTER TABLE ${DB}.${FULL_TABLE} DELETE WHERE timestamp between ${MIN_TS} and ${MAX_TS};

            ALTER TABLE ${DB}.${FULL_TABLE} ${ATTACH_PARTITIONS};

            DROP TABLE ${DB}.${TEMP_TABLE};""";

}
