package ru.yandex.webmaster3.worker.turbo.autoparser;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;

import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.BooleanNode;
import com.fasterxml.jackson.databind.node.TextNode;
import com.google.common.collect.Range;
import lombok.AllArgsConstructor;
import lombok.Getter;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.joda.time.DateTime;
import org.joda.time.LocalDate;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import ru.yandex.webmaster3.core.WebmasterException;
import ru.yandex.webmaster3.core.http.WebmasterErrorResponse;
import ru.yandex.webmaster3.core.util.environment.YandexEnvironmentProvider;
import ru.yandex.webmaster3.core.util.environment.YandexEnvironmentType;
import ru.yandex.webmaster3.core.worker.task.PeriodicTaskState;
import ru.yandex.webmaster3.core.worker.task.PeriodicTaskType;
import ru.yandex.webmaster3.core.worker.task.TaskResult;
import ru.yandex.webmaster3.storage.turbo.dao.autoparser.TurboAutoparserScreenshotsYDao;
import ru.yandex.webmaster3.storage.turbo.dao.autoparser.TurboAutoparserScreenshotsYDao.TurboAutoparserScreenshot;
import ru.yandex.webmaster3.storage.util.yt.AsyncTableReader;
import ru.yandex.webmaster3.storage.util.yt.YtCypressService;
import ru.yandex.webmaster3.storage.util.yt.YtNode;
import ru.yandex.webmaster3.storage.util.yt.YtPath;
import ru.yandex.webmaster3.storage.util.yt.YtService;
import ru.yandex.webmaster3.storage.util.yt.YtTableReadDriver;
import ru.yandex.webmaster3.storage.vanadium.VanadiumService;
import ru.yandex.webmaster3.storage.yql.YqlQueryBuilder;
import ru.yandex.webmaster3.storage.yql.YqlService;
import ru.yandex.webmaster3.worker.PeriodicTask;
import ru.yandex.webmaster3.worker.TaskSchedule;

/**
 * Created by Oleg Bazdyrev on 2019-08-09.
 */
@Slf4j
@Component
@RequiredArgsConstructor(onConstructor_ = @Autowired)
public class ProcessTurboAutoparserScreenshotsTask extends PeriodicTask<PeriodicTaskState> {

    private static final long TABLE_AGE_FOR_REMOVE = TimeUnit.DAYS.toMillis(7L);
    private static final String STATUS_SUCCESS = "success";
    private static final String TASK_PREFIX = "task-";
    private static final int SCHEENSHOT_WIDTH = 375;
    private static final int SCHEENSHOT_HEIGHT = 667;
    private static final String ATTR_VANADIUM_TASK_ID = "vanadium.taskid";
    private static final String ATTR_PROCESSED = "webmaster.processed.";

    private final TurboAutoparserScreenshotsYDao turboAutoparserScreenshotsYDao;
    private final VanadiumService vanadiumService;
    private final YtService ytService;
    private final YqlService yqlService;

    @Value("${webmaster3.worker.turbo.arnold.workDir}/urls-for-tellurium")
    private YtPath urlsForTelluriumTablePath;
    @Value("${webmaster3.worker.turbo.arnold.workDir}/queue-for-tellurium")
    private YtPath queueForTelluriumTablePath;
    @Value("${webmaster3.worker.turbo.arnold.workDir}/vanadium-in")
    private YtPath vanadiumInputPath;
    @Value("${webmaster3.worker.turbo.arnold.workDir}/vanadium-out")
    private YtPath vanadiumOutputPath;

    @Override
    public Result run(UUID runId) throws Exception {
        Map<YtPath, DateTime> processsedTables = processResultsFromTellurium();
        prepareTaskForTellurium(processsedTables);
        return new Result(TaskResult.SUCCESS);
    }

    private void prepareTaskForTellurium(Map<YtPath, DateTime> processedTables) throws Exception {
        // подготовим очередь
        ytService.inTransaction(queueForTelluriumTablePath).execute(cypressService -> {
            YqlQueryBuilder qb = new YqlQueryBuilder();
            qb.cluster(queueForTelluriumTablePath);
            qb.transaction(cypressService.getTransactionId());

            // подготовим урлы для обновления таймтемпов
            if (!processedTables.isEmpty()) {
                qb.appendText("$processedUrlDates = (SELECT Url, MAX(ProcessTimestamp) as ProcessTimestamp FROM (\n");
                boolean first = true;
                for (var entry : processedTables.entrySet()) {
                    if (!first) {
                        qb.appendText("UNION ALL\n");
                    }
                    first = false;
                    qb.appendText("SELECT url as Url, " + entry.getValue().getMillis() + " as ProcessTimestamp FROM");
                    qb.appendTable(entry.getKey()).appendText("\n");
                }
                qb.appendText(")\n GROUP BY Url);\n\n");
            }

            qb.appendText("INSERT INTO").appendTable(queueForTelluriumTablePath).appendText("WITH TRUNCATE\n");

            long minTimestamp = System.currentTimeMillis() - TimeUnit.DAYS.toMillis(20);
            // сделаем очередь, если ее еще нет
            if (!cypressService.exists(queueForTelluriumTablePath)) {
                qb.appendText("SELECT Url, TurboUrl, cast(null as Uint64) as ProcessTimestamp FROM");
                qb.appendTable(urlsForTelluriumTablePath).appendText("ORDER BY Url;\n\n");
            } else {
                // приджойним новые урлы и почистим старые, которых нет уже 15 дней
                qb.appendText("SELECT nvl(q.Url, u.Url) as Url, nvl(q.TurboUrl, u.TurboUrl) as TurboUrl, ProcessTimestamp\n");
                qb.appendText("FROM").appendTable(queueForTelluriumTablePath).appendText("as q\n");
                qb.appendText("FULL JOIN").appendTable(urlsForTelluriumTablePath).appendText("as u\n");
                qb.appendText("ON q.Url == u.Url\n");
                qb.appendText("WHERE u.Url is not null OR q.ProcessTimestamp > " + minTimestamp);
                qb.appendText("ORDER BY Url;\n\n");
            }
            qb.appendText("COMMIT;\n\n");

            if (!processedTables.isEmpty()) {
                qb.appendText("INSERT INTO").appendTable(queueForTelluriumTablePath).appendText("WITH TRUNCATE\n");
                qb.appendText("SELECT q.Url as Url, q.TurboUrl as TurboUrl,\n");
                qb.appendText("COALESCE(MIN_OF(ud.ProcessTimestamp, tud.ProcessTimestamp), q.ProcessTimestamp) as ProcessTimestamp\n");
                qb.appendText("FROM").appendTable(queueForTelluriumTablePath).appendText("as q\n");
                qb.appendText("LEFT JOIN $processedUrlDates as ud ON q.Url == ud.Url\n");
                qb.appendText("LEFT JOIN $processedUrlDates as tud ON q.TurboUrl == tud.Url;\n\n");
                qb.appendText("COMMIT;\n\n");
            }

            // если текущей дневной таблички с заданием еще нет
            String taskTableName = TASK_PREFIX + LocalDate.now().toString();
            YtPath taskTable = YtPath.path(vanadiumInputPath, taskTableName);
            boolean newTaskCreated = false;
            if (!cypressService.exists(taskTable)) {
                newTaskCreated = true;
                // переобойдем урлы старше 15 дней (но ограничим все 5000 * 2 урлов и отсортируем рандомно)
                qb.appendText("INSERT INTO").appendTable(taskTable).appendText("WITH TRUNCATE\n");
                qb.appendText("SELECT url FROM (\n");
                qb.appendText("SELECT AsList(Url, TurboUrl) as Urls FROM").appendTable(queueForTelluriumTablePath);
                qb.appendText("WHERE Url::Parse(Url).ParseError is null AND ProcessTimestamp is null OR ProcessTimestamp < " + minTimestamp + "\n");
                qb.appendText("ORDER BY Random(Urls) LIMIT 5000\n");
                qb.appendText(") FLATTEN BY Urls as url;");
            }

            if (YandexEnvironmentProvider.getEnvironmentType() == YandexEnvironmentType.PRODUCTION) {
                // только в проде делаем задания
                yqlService.execute(qb.build());
            } else {
                newTaskCreated = false;
                log.info("About to execute YQL: {}", qb.build());
            }

            if (newTaskCreated) {
                List<String> urls = new ArrayList<>();
                AsyncTableReader<Row> tableReader = new AsyncTableReader<>(cypressService, taskTable, Range.all(), YtTableReadDriver.createYSONDriver(Row.class))
                        .needLock(false);
                try (AsyncTableReader.TableIterator<Row> iterator = tableReader.read()) {
                    while (iterator.hasNext()) {
                        urls.add(iterator.next().getUrl());
                    }
                } catch (Exception e) {
                    throw new WebmasterException("Error reading urls table", new WebmasterErrorResponse.YTServiceErrorResponse(getClass(), e), e);
                }
                // отправим задание в Vanadium
                String taskId = vanadiumService.startScreenshotsTask(taskTable.getCluster(), urls, SCHEENSHOT_WIDTH, SCHEENSHOT_HEIGHT,
                        VanadiumService.Platform.TOUCH);
                // проставим параметры задания для Tellurium
                cypressService.set(YtPath.attribute(taskTable, ATTR_VANADIUM_TASK_ID), TextNode.valueOf(taskId));
            }

            // пометим таблицы, как обработанные
            String attrProcessed = ATTR_PROCESSED + YandexEnvironmentProvider.getEnvironmentType().name().toLowerCase();
            processedTables.keySet().forEach(path -> {
                cypressService.set(YtPath.path(path, "@" + attrProcessed), BooleanNode.TRUE);
            });

            return true;
        });
    }

    private Map<YtPath, DateTime> processResultsFromTellurium() throws Exception {
        return ytService.withoutTransactionQuery(cypressService -> {
            // создадим таблички на всякий пожарный
            cypressService.create(vanadiumInputPath, YtNode.NodeType.MAP_NODE, true, null, true);
            cypressService.create(vanadiumOutputPath, YtNode.NodeType.MAP_NODE, true, null, true);
            String attrProcessed = ATTR_PROCESSED + YandexEnvironmentProvider.getEnvironmentType().name().toLowerCase();
            Map<YtPath, DateTime> processedTables = new HashMap<>();
            // проверяем результаты
            for (YtPath inputTable : cypressService.list(vanadiumInputPath)) {
                YtNode node = cypressService.getNode(inputTable);
                if (node.getNodeMeta().has(attrProcessed)) {
                    if (System.currentTimeMillis() - node.getUpdateTime().getMillis() > TABLE_AGE_FOR_REMOVE) {
                        cypressService.remove(inputTable);
                    }
                    continue;
                }
                if (node.getNodeMeta().has(attrProcessed)) {
                    continue;
                }
                String taskId = node.getNodeMeta().get(ATTR_VANADIUM_TASK_ID).asText();
                VanadiumService.BatchStatusOutput batchStatus = vanadiumService.getBatchStatus(taskId);
                if (batchStatus.getBatchStatus() == VanadiumService.Status.SUCCEEDED) {
                    // копируем результаты от Vanadium
                    YtPath resultTable = YtPath.create(batchStatus.getYtCluster(), batchStatus.getBatchOutputTablePath());
                    cypressService.copy(resultTable, YtPath.path(vanadiumOutputPath, inputTable.getName()), true);
                    cypressService.set(YtPath.attribute(inputTable, attrProcessed), BooleanNode.TRUE);
                }
            }
            // ищем необработанные таблицы
            for (YtPath outputTable : cypressService.list(vanadiumOutputPath)) {
                YtNode node = cypressService.getNode(outputTable);
                if (node.getNodeMeta().has(attrProcessed)) {
                    if (System.currentTimeMillis() - node.getUpdateTime().getMillis() > TABLE_AGE_FOR_REMOVE) {
                        cypressService.remove(outputTable);
                    }
                    continue;
                }
                processTelluriumTable(cypressService, outputTable);
                processedTables.put(outputTable, node.getUpdateTime());
            }
            return processedTables;
        });
    }

    private void processTelluriumTable(YtCypressService cypressService, YtPath table) {
        AsyncTableReader<ScreenshotsRow> tableReader = new AsyncTableReader<>(cypressService, table, Range.all(),
                YtTableReadDriver.createYSONDriver(ScreenshotsRow.class)).withThreadName("tellurium-results-reader");
        try (var iterator = tableReader.read()) {
            List<TurboAutoparserScreenshot> screenshots = new ArrayList<>();
            while (iterator.hasNext()) {
                ScreenshotsRow row = iterator.next();
                if (row.isFinal() && STATUS_SUCCESS.equalsIgnoreCase(row.status)) {
                    screenshots.add(new TurboAutoparserScreenshot(row.getUrl(), row.getScreenshot(), row.getDetails(), row.getDuration(), DateTime.now()));
                    if (screenshots.size() >= 500) {
                        turboAutoparserScreenshotsYDao.addScreenshots(screenshots);
                        screenshots.clear();
                    }
                }
            }
            if (!screenshots.isEmpty()) {
                turboAutoparserScreenshotsYDao.addScreenshots(screenshots);
            }
        } catch (Exception e) {
            throw new WebmasterException("Yt error when reading table " + table,
                    new WebmasterErrorResponse.YTServiceErrorResponse(getClass(), e), e);
        }
    }

    @Override
    public PeriodicTaskType getType() {
        return PeriodicTaskType.PROCESS_TURBO_AUTOPARSER_SCREENSHOTS;
    }

    @Override
    public TaskSchedule getSchedule() {
        return TaskSchedule.startByCron("0 10 6/12 * * *");
    }

    @lombok.Value
    @AllArgsConstructor(onConstructor_ = @JsonCreator)
    @JsonIgnoreProperties(ignoreUnknown = true)
    private static final class Row {
        @JsonProperty("url")
        String url;
    }

    @Getter
    @AllArgsConstructor(onConstructor_ = {@JsonCreator})
    @JsonIgnoreProperties(ignoreUnknown = true)
    public static final class ScreenshotsRow {
        String url;
        String screenshot;
        String status;
        String failure;
        JsonNode meta;
        JsonNode details;
        JsonNode duration;
        boolean cached;
        @JsonProperty("final")
        boolean isFinal;
    }

}
