package ru.yandex.webmaster3.worker.niche;

import java.util.Collections;
import java.util.Map;

import org.apache.commons.lang3.text.StrSubstitutor;
import org.joda.time.Duration;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;

import ru.yandex.webmaster3.core.worker.task.PeriodicTaskType;
import ru.yandex.webmaster3.storage.clickhouse.TableType;
import ru.yandex.webmaster3.storage.util.clickhouse2.AbstractClickhouseDao;
import ru.yandex.webmaster3.storage.util.clickhouse2.CHPrimitiveType;
import ru.yandex.webmaster3.storage.util.clickhouse2.CHTable;
import ru.yandex.webmaster3.storage.util.clickhouse2.ClickhouseHost;
import ru.yandex.webmaster3.storage.util.clickhouse2.ClickhouseQueryContext;
import ru.yandex.webmaster3.storage.yql.YqlFunctions;
import ru.yandex.webmaster3.storage.yql.YqlQueryBuilder;
import ru.yandex.webmaster3.storage.ytimport.YtClickhouseDataLoad;
import ru.yandex.webmaster3.storage.ytimport.YtClickhouseDataLoadType;
import ru.yandex.webmaster3.worker.TaskSchedule;
import ru.yandex.webmaster3.worker.turbo.AbstractYqlPrepareImportTask;

import static ru.yandex.webmaster3.storage.niche.NicheRivalQueriesReportCHDao.F;

/**
 * ishalaru
 * 30.06.2021
 **/
public class ImportNicheRivalQueriesReportTask extends AbstractYqlPrepareImportTask {
    private static final int LINES_COUNT = 512;
    private static final String TABLE_NAME = "rival_queries_report3";
    private static final DateTimeFormatter PARTITION_MONTH_FORMAT = DateTimeFormat.forPattern("yyyyMM");
    private static final String MERGE_TABLE_QUERY = "" +
            "ALTER TABLE ${DB}.${FULL_TABLE} DELETE WHERE date = '${DATE}'\n\n" +
            "" +
            "ALTER TABLE ${DB}.${FULL_TABLE} ATTACH PARTITION '${MONTH}' FROM ${DB}.${TEMP_TABLE}\n\n" +
            "" +
            "DROP TABLE ${DB}.${TEMP_TABLE};";

    private static final CHTable TABLE = CHTable.builder()
            .database(AbstractClickhouseDao.DB_WEBMASTER3_NICHE)
            .name("tmp_" + TABLE_NAME + "_%s")
            .partitionBy("toYYYYMM(date)")
            .sharded(true)
            .field(F.DATE, CHPrimitiveType.Date)
            .keyField(F.HOST_ID, CHPrimitiveType.String)
            .keyField(F.REGION_ID, CHPrimitiveType.Int64)
            .keyField(F.DEVICE_TYPE, CHPrimitiveType.UInt8)
            .field(F.RIVAL, CHPrimitiveType.String)
            .field(F.POPULARITY, CHPrimitiveType.Float64)
            .field(F.QUERY_SOURCE, CHPrimitiveType.UInt8)
            .build();

    @Override
    protected YtClickhouseDataLoad init(YtClickhouseDataLoad imprt) throws Exception {
        return initByTableDateName(imprt);
    }

    @Override
    protected int getShardsCount() {
        return clickhouseServer.getShardsCount();
    }

    //`Date`,`Host`,`Query`,`RegionId`,`IsMobile`,`IsPad`,`Count`,`MainHost`,`Position`,`QuerySource`
    @Override
    protected YqlQueryBuilder prepareIntermediateTable(YtClickhouseDataLoad imprt) {
        String tableDate = imprt.getSourceTable().getName();
        YqlQueryBuilder yqlQueryBuilder = YqlQueryBuilder.newBuilder()
                .cluster(tablePath)
                .appendFDefinition(YqlFunctions.URL_2_HOST_ID)
                .appendText("PRAGMA yt.MaxRowWeight = '128M';\n")
                .appendText("PRAGMA yt.DefaultMemoryLimit = '16G';\n\n")
                .appendText("INSERT INTO " + INTERMEDIATE_TABLE)
                .appendText("SELECT ShardId, RowId, Compress::Gzip(String::JoinFromList(AGGREGATE_LIST(data), ''), 6) as data FROM (\n")
                .appendText("SELECT (Digest::Fnv64($url2HostId(Host)) % " + getShardsCount() + ") as ShardId,")
                .appendText("((Digest::Fnv64($url2HostId(Host) || nvl($url2HostId(Rival), '')) / " + getShardsCount() + ") % " + LINES_COUNT + ") as RowId,")
                .appendText("'" + tableDate + "' || '\\t' ||\n")
                .appendText("($url2HostId(Host) || '\\t' ||\n")
                .appendText("CAST(RegionId as String) || '\\t' ||\n")
                .appendText("CAST(IF(IsMobile,if (IsPad,3,1),2) as String) || '\\t' ||\n")
                .appendText("nvl($url2HostId(Rival), '') || '\\t' ||\n")
                .appendText("CAST(Popularity as String) || '\\t' ||\n")
                .appendText("CAST(QuerySource as String) || '\\n') as data\n")
                .appendText("FROM (\n")
                .appendText("SELECT Host, Rival, -1 as RegionId, IsMobile, IsPad, QuerySource, sum(Popularity) as Popularity FROM ")
                .appendTable(imprt.getSourceTable()).appendText("\n")
                .appendText("GROUP BY Host, Rival, IsMobile, IsPad, QuerySource\n")
                .appendText("UNION ALL\n")
                .appendText("SELECT Host, Rival, RegionId, IsMobile, IsPad, QuerySource, Popularity FROM\n")
                .appendTable(imprt.getSourceTable()).appendText("\n")
                .appendText(")) \n GROUP BY ShardId, RowId;")
                .appendText("COMMIT;\n\n");

        return yqlQueryBuilder;
    }

    @Override
    protected YtClickhouseDataLoad rename(YtClickhouseDataLoad imprt) throws Exception {
        String database = getTable().getDatabase();
        String tempTableName = getTable().replicatedMergeTreeTableName(-1, imprt.getData());
        // удаляем из общей таблицы подливаемую дату и вливаем свежие данные
        for (ClickhouseHost host : clickhouseServer.getHosts()) {
            boolean hasTable = !clickhouseSystemTablesCHDao.getTables(host, database, Collections.singleton(tempTableName)).isEmpty();
            if (!hasTable) {
                continue;
            }
            // получим минимальный и максимальный ts в новой таблице
            ClickhouseQueryContext.Builder ctx = ClickhouseQueryContext.useDefaults().setHost(host).setTimeout(Duration.standardMinutes(10L));
            StrSubstitutor substitutor = new StrSubstitutor(Map.of(
                    "DB", database,
                    "FULL_TABLE", TABLE_NAME,
                    "TEMP_TABLE", tempTableName,
                    "DATE", imprt.getDateFrom().toString(),
                    "MONTH", PARTITION_MONTH_FORMAT.print(imprt.getDateFrom()),
                    "YEAR", String.valueOf(imprt.getDateFrom().getYear())
            ));
            for (String query : substitutor.replace(MERGE_TABLE_QUERY).split("\n\n")) {
                clickhouseServer.execute(ctx, query);
            }
        }

        return imprt.withNextState();
    }

    protected YtClickhouseDataLoad replicate(YtClickhouseDataLoad imprt) throws Exception {
        // no replication needed
        return imprt.withNextState();
    }

    @Override
    protected CHTable getTable() {
        return TABLE;
    }

    @Override
    protected TableType getTableType() {
        return TableType.NICHE_RIVAL_QUERIES_REPORT;
    }

    @Override
    protected YtClickhouseDataLoadType getImportType() {
        return YtClickhouseDataLoadType.NICHE_RIVAL_QUERIES_REPORT2;
    }

    @Override
    public TaskSchedule getSchedule() {
        return TaskSchedule.startByCron("0 */5 * * * *");
    }

    @Override
    public PeriodicTaskType getType() {
        return PeriodicTaskType.IMPORT_NICHE_RIVAL_QUERIES_REPORT;
    }
}
