package ru.yandex.webmaster3.worker.checklist;

import java.io.IOException;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.stream.Stream;

import com.datastax.driver.core.utils.UUIDs;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.node.TextNode;
import com.google.common.collect.Range;
import lombok.AllArgsConstructor;
import lombok.Setter;
import lombok.Value;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import org.springframework.beans.factory.annotation.Autowired;

import ru.yandex.webmaster3.core.WebmasterException;
import ru.yandex.webmaster3.core.checklist.data.SiteProblemContent.TooManyDomainsOnSearch;
import ru.yandex.webmaster3.core.checklist.data.SiteProblemTypeEnum;
import ru.yandex.webmaster3.core.data.WebmasterHostId;
import ru.yandex.webmaster3.core.http.WebmasterErrorResponse;
import ru.yandex.webmaster3.core.notification.LanguageEnum;
import ru.yandex.webmaster3.core.util.IdUtils;
import ru.yandex.webmaster3.core.util.environment.YandexEnvironmentProvider;
import ru.yandex.webmaster3.core.util.environment.YandexEnvironmentType;
import ru.yandex.webmaster3.core.worker.task.PeriodicTaskType;
import ru.yandex.webmaster3.storage.checklist.dao.MdbDomainsOnSearchCHDao;
import ru.yandex.webmaster3.storage.checklist.data.ProblemSignal;
import ru.yandex.webmaster3.storage.checklist.service.SiteProblemsService;
import ru.yandex.webmaster3.storage.clickhouse.TableType;
import ru.yandex.webmaster3.storage.host.AllHostsCacheService;
import ru.yandex.webmaster3.storage.host.CommonDataState;
import ru.yandex.webmaster3.storage.host.CommonDataType;
import ru.yandex.webmaster3.storage.notifications.NotificationChannel;
import ru.yandex.webmaster3.storage.settings.SettingsService;
import ru.yandex.webmaster3.storage.user.UserPersonalInfo;
import ru.yandex.webmaster3.storage.user.message.content.MessageContent;
import ru.yandex.webmaster3.storage.util.clickhouse2.CHTable;
import ru.yandex.webmaster3.storage.util.yt.AsyncTableReader;
import ru.yandex.webmaster3.storage.util.yt.YtCypressService;
import ru.yandex.webmaster3.storage.util.yt.YtPath;
import ru.yandex.webmaster3.storage.util.yt.YtTableReadDriver;
import ru.yandex.webmaster3.storage.yql.YqlQueryBuilder;
import ru.yandex.webmaster3.storage.ytimport.YtClickhouseDataLoad;
import ru.yandex.webmaster3.storage.ytimport.YtClickhouseDataLoadType;
import ru.yandex.webmaster3.worker.TaskSchedule;
import ru.yandex.webmaster3.worker.notifications.auto.AutoNotificationsSenderService;
import ru.yandex.webmaster3.worker.notifications.auto.NotificationInfo;
import ru.yandex.webmaster3.worker.turbo.AbstractYqlPrepareImportTask;

/**
 * Created by Oleg Bazdyrev on 25/03/2021.
 */
public class MdbImportDomainsOnSearch extends AbstractYqlPrepareImportTask {

    private static final DateTimeFormatter TABLE_NAME_PATTERN = DateTimeFormat.forPattern("yyyyMMdd-HHmmss");
    private static final Duration CHANGES_TABLE_MAX_AGE = Duration.standardDays(30L);
    private static final long PROBLEMS_BATCH_SIZE = 100;
    private static final YtPath JUPITER_YT_DIR = YtPath.create("arnold", "//home/jupiter");
    private static final String SEARCHBASE_DATE_ATTR = "@searchbase_prod";

    @Setter
    private AllHostsCacheService allHostsCacheService;
    @Setter
    private AutoNotificationsSenderService autoNotificationsSenderService;
    @Setter
    private SettingsService settingsService;
    @Autowired
    private SiteProblemsService siteProblemsService;
    @Setter
    private YtPath changesDir;
    @Setter
    private YtPath problemsTable;

    @Override
    protected YtClickhouseDataLoad init(YtClickhouseDataLoad imprt) throws Exception {
        if (YandexEnvironmentProvider.getEnvironmentType() == YandexEnvironmentType.PRODUCTION) {
            updateSearchbaseDate();
        }
        return initByMaxTableName(imprt);
    }

    @Override
    protected YqlQueryBuilder prepareIntermediateTable(YtClickhouseDataLoad imprt) {
        YqlQueryBuilder queryBuilder = new YqlQueryBuilder();
        queryBuilder
                .cluster(tablePath)
                .inferSchema(YqlQueryBuilder.InferSchemaMode.INFER)
                .appendText("PRAGMA yt.MaxRowWeight = '128M';\n")
                .appendText("PRAGMA yt.DefaultMemoryLimit = '4G';\n\n")
                .appendText("INSERT INTO " + INTERMEDIATE_TABLE)
                .appendText("SELECT ShardId, RowId, Compress::Gzip(String::JoinFromList(AGGREGATE_LIST(data), ''), 6) as data FROM\n")
                .appendText("(\n")
                .appendText("  SELECT\n")
                .appendText("    (Digest::Fnv64(Owner) % " + getShardsCount() + ") as ShardId,\n")
                .appendText("    (Digest::CityHash(Owner || Domain) % " + 256 + ") as RowId,\n")
                .appendText("    (\n")
                .appendText("      Owner  || '\\t' ||\n")
                .appendText("      Domain || '\\n'\n")
                .appendText("    ) as data\n")
                .appendText("  FROM").appendTable(imprt.getSourceTable()).appendText("\n")
                .appendText(")\n")
                .appendText("GROUP BY ShardId, RowId;\n\n")
                .appendText("INSERT INTO").appendTable(problemsTable).appendText("WITH TRUNCATE\n")
                .appendText("SELECT Owner, AGGREGATE_LIST(Domain, 10) as Domains\n")
                .appendText("FROM").appendTable(imprt.getSourceTable()).appendText("\n")
                .appendText("GROUP BY Owner\n")
                .appendText("HAVING count(*) >= " + TooManyDomainsOnSearch.THRESHOLD + ";\n")
                .appendText("COMMIT;\n\n");

        return queryBuilder;
    }

    @Override
    protected YtClickhouseDataLoad rename(YtClickhouseDataLoad imprt) throws Exception {
        YtClickhouseDataLoad result = super.rename(imprt);
        ytService.withoutTransaction(cypressService -> {
            updateProblems(cypressService);
            String lastProcessedTable = Optional.ofNullable(settingsService.getSettingUncached(CommonDataType.LAST_IMPORTED_DOMAINS_ON_SEARCH_CHANGES))
                    .map(CommonDataState::getValue).orElse("");
            List<YtPath> tablesToProcess = cypressService.list(changesDir);
            tablesToProcess.sort(null);
            log.info("Found {} changes tables to process: {}", tablesToProcess.size(), tablesToProcess);
            for (YtPath table : tablesToProcess) {
                String tableName = table.getName().replace("-", "");
                if (tableName.compareTo(imprt.getData()) > 0) { // too fresh
                    break;
                }
                if (tableName.compareTo(lastProcessedTable) <= 0) { // too old
                    continue;
                }
                // process notifications
                processChanges(cypressService, table);

                lastProcessedTable = tableName;
                settingsService.update(CommonDataType.LAST_IMPORTED_DOMAINS_ON_SEARCH_CHANGES, lastProcessedTable);
            }
            cleanup(cypressService);
            return true;
        });

        return result;
    }

    /**
     * Обновляет проблемы из таблички problems
     * @param cypressService
     */
    private void updateProblems(YtCypressService cypressService) throws InterruptedException {
        DateTime updateStartTime = DateTime.now();
        AsyncTableReader<ProblemRow> tableReader = new AsyncTableReader<>(cypressService, problemsTable, Range.all(),
                YtTableReadDriver.createYSONDriver(ProblemRow.class)).withThreadName("too-many-domains-on-search-reader");
        try (var iterator = tableReader.read()) {
            Map<WebmasterHostId, ProblemSignal> problemsByHostId = new HashMap<>();
            while (iterator.hasNext()) {
                ProblemRow row = iterator.next();
                WebmasterHostId httpHostId = IdUtils.urlToHostId(WebmasterHostId.Schema.HTTP.getSchemaPrefix() + row.owner);
                WebmasterHostId httpsHostId = IdUtils.urlToHostId(WebmasterHostId.Schema.HTTPS.getSchemaPrefix() + row.owner);
                Stream.of(httpHostId, httpsHostId)
                        .filter(allHostsCacheService::contains)
                        .forEach(h -> problemsByHostId.put(h, new ProblemSignal(new TooManyDomainsOnSearch(row.domains), DateTime.now())));
                if (problemsByHostId.size() > PROBLEMS_BATCH_SIZE) {
                    log.info("Storing TOO_MANY_DOMAINS_ON_SEARCH problems for {} hosts", problemsByHostId.size());
                    siteProblemsService.updateCleanableProblems(problemsByHostId, SiteProblemTypeEnum.TOO_MANY_DOMAINS_ON_SEARCH);
                    problemsByHostId.clear();
                }
            }
            if (!problemsByHostId.isEmpty()) {
                log.info("Storing TOO_MANY_DOMAINS_ON_SEARCH problems for {} hosts", problemsByHostId.size());
                siteProblemsService.updateCleanableProblems(problemsByHostId, SiteProblemTypeEnum.TOO_MANY_DOMAINS_ON_SEARCH);
            }
            siteProblemsService.notifyCleanableProblemUpdateFinished(SiteProblemTypeEnum.TOO_MANY_DOMAINS_ON_SEARCH, updateStartTime);
        } catch (IOException e) {
            throw new WebmasterException("Error reading table", new WebmasterErrorResponse.YTServiceErrorResponse(getClass(), e), e);
        }
    }

    private void processChanges(YtCypressService cypressService, YtPath table) throws InterruptedException {
        log.info("Processing changes table {}", table);
        DateTime now = DateTime.now();
        AsyncTableReader<NewDomainsRow> tableReader = new AsyncTableReader<>(cypressService, table, Range.all(),
                YtTableReadDriver.createYSONDriver(NewDomainsRow.class)).withThreadName("new-domains-notification-reader");
        try (var iterator = tableReader.read()) {
            while (iterator.hasNext()) {
                NewDomainsRow row = iterator.next();
                for (NotificationChannel channel : row.getChannels()) {
                    NotificationInfo notificationInfo = NotificationInfo.builder()
                            .id(UUIDs.timeBased())
                            .email(row.getEmail())
                            .hostId(row.getHostId())
                            .userId(row.getUserId())
                            .personalInfo(new UserPersonalInfo(row.getUserId(), row.getLogin(), row.getFio(), row.getLanguage()))
                            .messageContent(row.getContent())
                            .channel(channel)
                            .critical(false)
                            .build();
                    log.info("Sending notification for host {} to user {} by channel {}, email - {}", row.getHostId(), row.getUserId(), channel, row.getEmail());
                    autoNotificationsSenderService.sendMessage(notificationInfo);
                }
            }
        } catch (IOException e) {
            throw new WebmasterException("Error reading table", new WebmasterErrorResponse.YTServiceErrorResponse(getClass(), e), e);
        }
    }

    private void cleanup(YtCypressService cypressService) {
        DateTime maxTableAge = DateTime.now().minus(CHANGES_TABLE_MAX_AGE);
        for (YtPath table : cypressService.list(changesDir)) {
            DateTime tableDate = TABLE_NAME_PATTERN.parseDateTime(table.getName());
            if (tableDate.isBefore(maxTableAge)) {
                log.info("Removing old table {}", table);
                if (YandexEnvironmentProvider.getEnvironmentType() == YandexEnvironmentType.PRODUCTION) {
                    cypressService.remove(table);
                }
            }
        }
    }

    // Сейчас чтение даты поисковой базы из атрибутов //home/jupiter в YQL сломано
    // Поэтому здесь мы читаем этот атрибут и пишем его в директорию, из которой можно будет безопасно его прочитать
    private void updateSearchbaseDate() throws Exception {
        ytService.withoutTransaction(cypressService -> {
            String searchBaseDate = Optional.of(cypressService.getNode(JUPITER_YT_DIR))
                    .map(n -> n.getNodeMeta())
                    .map(n -> n.get("jupiter_meta"))
                    .map(n -> n.get("production_current_state"))
                    .map(n -> n.asText())
                    .orElseThrow(() -> new IllegalStateException("Expected to find searchbase date"));
            cypressService.set(YtPath.path(tablePath, SEARCHBASE_DATE_ATTR), TextNode.valueOf(searchBaseDate));
            return true;
        });
    }

    @Override
    protected CHTable getTable() {
        return MdbDomainsOnSearchCHDao.TABLE;
    }

    @Override
    protected TableType getTableType() {
        return TableType.DOMAINS_ON_SEARCH;
    }

    @Override
    protected YtClickhouseDataLoadType getImportType() {
        return YtClickhouseDataLoadType.DOMAINS_ON_SEARCH;
    }

    @Override
    public PeriodicTaskType getType() {
        return PeriodicTaskType.MDB_IMPORT_DOMAINS_ON_SEARCH;
    }

    @Override
    public TaskSchedule getSchedule() {
        return TaskSchedule.startByCron("0 37 * * * *");
    }

    @Value
    @AllArgsConstructor(onConstructor_ = @JsonCreator)
    private static class NewDomainsRow {
        @JsonProperty("host_id")
        WebmasterHostId hostId;
        @JsonProperty("channel_email")
        boolean channelEmail;
        @JsonProperty("channel_service")
        boolean channelService;
        @JsonProperty("domains")
        List<String> domains;
        @JsonProperty("user_id")
        long userId;
        @JsonProperty("email")
        String email;
        @JsonProperty("fio")
        String fio;
        @JsonProperty("language")
        LanguageEnum language;
        @JsonProperty("login")
        String login;

        @JsonIgnore
        public MessageContent.NewDomainsNotification getContent() {
            return new MessageContent.NewDomainsNotification(hostId, domains);
        }

        public Set<NotificationChannel> getChannels() {
            EnumSet<NotificationChannel> channels = EnumSet.noneOf(NotificationChannel.class);
            if (channelEmail) {
                channels.add(NotificationChannel.EMAIL);
            }
            if (channelService) {
                channels.add(NotificationChannel.SERVICE);
            }
            return channels;
        }
    }

    @Value
    @AllArgsConstructor(onConstructor_ = @JsonCreator)
    private static class ProblemRow {
        @JsonProperty("Owner")
        String owner;
        @JsonProperty("Domains")
        List<String> domains;
    }


}
