package ru.yandex.webmaster3.worker.turbo.autoparser;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.module.paramnames.ParameterNamesModule;
import com.google.common.collect.Range;
import lombok.RequiredArgsConstructor;
import org.joda.time.DateTime;
import org.joda.time.format.DateTimeFormat;
import org.joda.time.format.DateTimeFormatter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import ru.yandex.webmaster3.core.WebmasterException;
import ru.yandex.webmaster3.core.data.WebmasterHostId;
import ru.yandex.webmaster3.core.host.service.HostOwnerService;
import ru.yandex.webmaster3.core.http.WebmasterErrorResponse;
import ru.yandex.webmaster3.core.turbo.model.TurboSampleData;
import ru.yandex.webmaster3.core.turbo.model.autoparser.TurboAutoparsedHostInfo;
import ru.yandex.webmaster3.core.util.IdUtils;
import ru.yandex.webmaster3.core.worker.task.PeriodicTaskState;
import ru.yandex.webmaster3.core.worker.task.PeriodicTaskType;
import ru.yandex.webmaster3.core.worker.task.TaskResult;
import ru.yandex.webmaster3.storage.host.CommonDataState;
import ru.yandex.webmaster3.storage.host.CommonDataType;
import ru.yandex.webmaster3.storage.settings.SettingsService;
import ru.yandex.webmaster3.storage.turbo.dao.autoparser.TurboAutoparsedYDao;
import ru.yandex.webmaster3.storage.turbo.service.autoparser.TurboAutoparserInfoUtil;
import ru.yandex.webmaster3.storage.turbo.service.settings.TurboSettingsService;
import ru.yandex.webmaster3.storage.util.yt.AsyncTableReader;
import ru.yandex.webmaster3.storage.util.yt.YtCypressService;
import ru.yandex.webmaster3.storage.util.yt.YtException;
import ru.yandex.webmaster3.storage.util.yt.YtNode;
import ru.yandex.webmaster3.storage.util.yt.YtPath;
import ru.yandex.webmaster3.storage.util.yt.YtService;
import ru.yandex.webmaster3.storage.util.yt.YtTableReadDriver;
import ru.yandex.webmaster3.storage.util.yt.transfer.YtTransferManager;
import ru.yandex.webmaster3.storage.yql.YqlQueryBuilder;
import ru.yandex.webmaster3.storage.yql.YqlService;
import ru.yandex.webmaster3.worker.PeriodicTask;
import ru.yandex.webmaster3.worker.TaskSchedule;

/**
 * Created by ifilippov5 on 18.06.18.
 */
@Component("importTurboAutoparsedHostsTask")
@RequiredArgsConstructor(onConstructor_ = @Autowired)
public class ImportTurboAutoparsedHostsTask extends PeriodicTask<PeriodicTaskState> {
    private static final Logger log = LoggerFactory.getLogger(ImportTurboAutoparsedHostsTask.class);

    private static final ObjectMapper OM = new ObjectMapper()
            .registerModule(new ParameterNamesModule())
            .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
    private static final String ATTR_LAST_PROCESSED = "modification_time";
    private static final DateTimeFormatter modificationDateTimeFormat = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'");

    private final SettingsService settingsService;
    private final TurboAutoparsedYDao turboAutoparsedYDao;
    private final TurboSettingsService turboSettingsService;
    private final HostOwnerService hostOwnerService;
    private final YtService ytService;
    private final YtTransferManager ytTransferManager;
    private final YqlService yqlService;
    @Value("${webmaster3.worker.turbo.autoparser.import.tablePath}")
    private final YtPath tablePath;
    @Value("${webmaster3.worker.turbo.autoparser.import.workDir}/history")
    private final YtPath historyTablePath;
    @Value("${webmaster3.worker.turbo.autoparser.import.workDir}/urls-for-tellurium")
    private final YtPath urlsForTelluriumTablePath;
    @Value("${webmaster3.worker.turbo.arnold.workDir}/urls-for-tellurium")
    private final YtPath urlsForTelluriumArnoldTablePath;
    @Value("${webmaster3.worker.turbo.autoparser.export.screenshots.tablePath}")
    private final YtPath screenshotsTablePath;

    @Override
    public Result run(UUID runId) throws Exception {
        CommonDataState state = settingsService.getSettingUncached(getCommonDataType());
        DateTime lastImportDate = (state == null ? null : DateTime.parse(state.getValue()));
        DateTime tableLastUpdateDate = getDataUpdateTimeForTableOnYT();
        Collection<WebmasterHostId> hostsWaitNotification = new HashSet<>();
        if (tableLastUpdateDate != null && (lastImportDate == null || lastImportDate.isBefore(tableLastUpdateDate))) {
            Map<String, TurboAutoparsedHostInfo> dataFromYt = new HashMap<>();
            ytService.inTransaction(tablePath).execute(cypressService -> {
                process(cypressService, tableLastUpdateDate, dataFromYt);
                return true;
            });
            Map<String, String> domains2Owners = hostOwnerService.mapDomainsToOwners(dataFromYt.keySet());
            Map<String, TurboAutoparsedHostInfo> data = new HashMap<>();

            //актуализируем значения галок, если уже было какое-то состояние, то оставляем, иначе по умолчанию
            for (String domain : dataFromYt.keySet()) {
                TurboAutoparsedHostInfo oldInfo = turboAutoparsedYDao.get(domain);
                if (oldInfo != null && oldInfo.getCheckboxState() != null) {
                    data.put(domain, dataFromYt.get(domain).withCheckboxState(oldInfo.getCheckboxState()));
                } else {
                    data.put(domain, dataFromYt.get(domain));
                    if (oldInfo == null) {
                        hostsWaitNotification.add(IdUtils.urlToHostId(domain));
                    }
                }
                List<String> tldsChain = TurboAutoparserInfoUtil.getParentsChain(domain, domains2Owners.get(domain));
                for (String tld : tldsChain) {
                    domains2Owners.put(tld, domains2Owners.get(domain));
                    if (data.containsKey(tld) || dataFromYt.containsKey(tld)) {
                        continue;
                    }
                    TurboAutoparsedHostInfo tldInfo = turboAutoparsedYDao.get(tld);
                    TurboAutoparsedHostInfo newTldInfo = new TurboAutoparsedHostInfo(tld, TurboAutoparserInfoUtil.DEFAULT_AUTOPARSER_CHECKBOX_STATE,
                            new ArrayList<>(), tableLastUpdateDate, false);//старые сэмплы из прошлых импортом удаляем
                    if (tldInfo != null && tldInfo.getCheckboxState() != null) {
                        data.put(tld, newTldInfo.withCheckboxState(tldInfo.getCheckboxState()));//а вот состояние галки сохраняется
                    } else {
                        data.put(tld, newTldInfo);
                        if (tldInfo == null) {
                            hostsWaitNotification.add(IdUtils.urlToHostId(domain));
                        }
                    }
                }
            }

            TurboAutoparserInfoUtil.pushUpAutoparsedSamples(data, domains2Owners);
            turboAutoparsedYDao.insert(data.values());
            settingsService.update(getCommonDataType(), String.valueOf(tableLastUpdateDate));
            for (TurboAutoparsedHostInfo info : data.values()) {
                turboSettingsService.notifyAboutSettingsChange(info.getHost(), null, info.getCheckboxState(), null);
            }
        } else {
            log.info("Table on YT not modified");
        }
        return new Result(TaskResult.SUCCESS);
    }

    private void process(YtCypressService cypressService, DateTime tableLastUpdateDate, Map<String, TurboAutoparsedHostInfo> data) throws YtException {
        ExecutorService executorService = Executors.newFixedThreadPool(1);
        try {
            // сохраним для истории
            saveHistory(cypressService, tableLastUpdateDate);
            readYtTable(executorService, cypressService, tablePath, Range.all(), tableLastUpdateDate, data);
        } finally {
            executorService.shutdown();
        }
    }

    private void saveHistory(YtCypressService cypressService, DateTime tableLastUpdateDate) {
        // подготовим запрос
        YqlQueryBuilder qb = new YqlQueryBuilder();
        qb.appendText("insert into ").appendTable(historyTablePath);
        qb.appendText("select ").appendText(String.valueOf(tableLastUpdateDate.getMillis())).appendText(" as `Timestamp`, ");
        qb.appendText("Host, Samples from ").appendTable(tablePath).appendText(";\n\n");

        qb.appendText("insert into ").appendTable(screenshotsTablePath).appendText(" with truncate");
        qb.appendText("select Host, Url from (\n");
        qb.appendText("select Host, ListFlatMap(Yson::ConvertToList(Samples),");
        qb.appendText("($sample) -> { \n" +
                "return AsList(Yson::ConvertToString($sample.TurboUrl) || '&noredirect=1', Yson::ConvertToString($sample.Url)); \n" +
                "}) as Urls");
        qb.appendText("from").appendTable(tablePath).appendText(")\n");
        qb.appendText("flatten by Urls as Url order by Host, Url;\n");

        // подготовим список урлов (пар url-turboUrl) для прогрузки через Tellurium
        qb.appendText("insert into ").appendTable(urlsForTelluriumTablePath).appendText(" with truncate");
        qb.appendText("select Url, some(UrlPair.1) as TurboUrl from (\n");
        qb.appendText("select ListMap(Yson::ConvertToList(Samples),");
        qb.appendText("($sample) -> { \n" +
                "return AsTuple(Yson::ConvertToString($sample.Url), Yson::ConvertToString($sample.TurboUrl) || '&noredirect=1'); \n" +
                "}) as UrlPairs");
        qb.appendText("from").appendTable(tablePath).appendText(")\n");
        qb.appendText("flatten by UrlPairs as UrlPair group by UrlPair.0 as Url order by Url;\n");

        // выполняем
        yqlService.execute(qb.build());

        // копируем на арнольд задания
        String transferTaskId = ytTransferManager.addTask(urlsForTelluriumTablePath, urlsForTelluriumArnoldTablePath);
        log.info("Added transfer manager task {}", transferTaskId);
    }

    private void readYtTable(ExecutorService executorService, YtCypressService cypressService, YtPath tablePath, Range<Long> range,
                             DateTime tableLastUpdateDate, Map<String, TurboAutoparsedHostInfo> data) throws YtException {
        AsyncTableReader<YtRow> tableReader = new AsyncTableReader<>(cypressService, tablePath, range,
                YtTableReadDriver.createYSONDriver(YtRow.class, OM)).splitInParts(10000L)
                .inExecutor(executorService, "turbo-autoparser-cacher")
                .withRetry(5);

        try (AsyncTableReader.TableIterator<YtRow> it = tableReader.read()) {
            while (it.hasNext()) {
                YtRow row = it.next();

                List<TurboSampleData> samples = row.samples;
                if (samples.size() > TurboAutoparserInfoUtil.MAX_SAMPLES_COUNT) {
                    samples = samples.subList(0, TurboAutoparserInfoUtil.MAX_SAMPLES_COUNT);
                }
                data.put(row.host, new TurboAutoparsedHostInfo(row.host, TurboAutoparserInfoUtil.DEFAULT_AUTOPARSER_CHECKBOX_STATE,
                        samples, tableLastUpdateDate, true));
            }
        } catch (IOException | InterruptedException | YtException e) {
            throw new YtException("Unable to read table: " + tablePath, e);
        } catch (Exception e) {
            throw new WebmasterException("Unknown error",
                    new WebmasterErrorResponse.InternalUnknownErrorResponse(getClass(), "Unknown error"), e);
        }
    }

    private DateTime getDataUpdateTimeForTableOnYT() throws YtException, InterruptedException {
        return ytService.withoutTransactionQuery(cypressService -> {
            if (!cypressService.exists(tablePath)) {
                return null;
            }
            YtNode result = cypressService.getNode(tablePath);
            String modificationTimeAsText = result.getNodeMeta().get(ATTR_LAST_PROCESSED).asText();
            return DateTime.parse(modificationTimeAsText, modificationDateTimeFormat);
        });
    }

    protected CommonDataType getCommonDataType() {
        return CommonDataType.TURBO_AUTOPARSED_HOSTS_LAST_IMPORT_DATE;
    }

    @Override
    public PeriodicTaskType getType() {
        return PeriodicTaskType.IMPORT_TURBO_AUTOPARSED_HOSTS;
    }

    @Override
    public TaskSchedule getSchedule() {
        return TaskSchedule.startByCron("0 0 * * * *");
    }

    public static class YtRow {
        private final String host;
        private final List<TurboSampleData> samples;

        @JsonCreator
        public YtRow(String host, List<TurboSampleData> samples) {
            this.host = host;
            this.samples = samples;
        }

        @JsonProperty("Host")
        public String getHost() {
            return host;
        }

        @JsonProperty("Samples")
        public List<TurboSampleData> getSamples() {
            return samples;
        }
    }
}
