package ru.yandex.webmaster3.importer.service;

import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.stream.Collectors;

import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory;
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
import com.fasterxml.jackson.datatype.joda.JodaModule;
import com.fasterxml.jackson.module.paramnames.ParameterNamesModule;
import com.google.common.collect.Sets;
import lombok.AllArgsConstructor;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.IOUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.PropertySourcesPlaceholderConfigurer;
import org.springframework.core.env.PropertySourcesPropertyResolver;
import org.springframework.core.io.Resource;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.scheduling.concurrent.CustomizableThreadFactory;
import org.springframework.stereotype.Service;

import ru.yandex.webmaster3.core.http.WebmasterJsonModule;
import ru.yandex.webmaster3.core.util.joda.jackson.WebmasterDurationModule;
import ru.yandex.webmaster3.core.util.json.JsonMapping;
import ru.yandex.webmaster3.storage.importer.dao.ClickhouseImportDefinitionsYDao;
import ru.yandex.webmaster3.storage.importer.dao.ClickhouseImportTasksYDao;
import ru.yandex.webmaster3.storage.importer.model.ImportContext;
import ru.yandex.webmaster3.storage.importer.model.ImportDefinition;
import ru.yandex.webmaster3.storage.importer.model.ImportPolicy;
import ru.yandex.webmaster3.storage.importer.model.ImportStage;
import ru.yandex.webmaster3.storage.importer.model.ImportTask;
import ru.yandex.webmaster3.storage.importer.model.importing.ImportWithTransferManager;
import ru.yandex.webmaster3.storage.importer.model.switching.ImportSwitchSearchQueries;
import ru.yandex.webmaster3.storage.notifications.service.EmailSenderService;
import ru.yandex.webmaster3.storage.util.yt.YtCypressService;
import ru.yandex.webmaster3.storage.util.yt.YtException;
import ru.yandex.webmaster3.storage.util.yt.YtLockMode;
import ru.yandex.webmaster3.storage.util.yt.YtNode;
import ru.yandex.webmaster3.storage.util.yt.YtPath;
import ru.yandex.webmaster3.storage.util.yt.YtService;

/**
 * Created by Oleg Bazdyrev on 20/09/2020.
 */
@Slf4j
@Service
@RequiredArgsConstructor(onConstructor_ = @Autowired)
public class ImporterTaskRunner {

    private static final ObjectMapper YAML_OM = new ObjectMapper(new YAMLFactory())
            .registerModule(new WebmasterJsonModule(false))
            .registerModule(new JodaModule())
            .registerModule(new ParameterNamesModule())
            .registerModule(new WebmasterDurationModule(true))
            .registerModule(new Jdk8Module())
            .setSerializationInclusion(JsonInclude.Include.NON_NULL)
            .disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES)
            .disable(SerializationFeature.FAIL_ON_EMPTY_BEANS)
            .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS)
            .enable(DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_USING_DEFAULT_VALUE);

    private static final int WORKER_COUNT = 5;
    private static final long DEFAULT_DELAY = TimeUnit.MINUTES.toMillis(1L);
    private static final Set<String> INTERRUPTED_MESSAGES = Sets.newHashSet("Interrupted!", "Yql exception, code: 0, host: yql.yandex.net, port: 443; sleep interrupted");

    private final ApplicationContext applicationContext;
    private final PropertySourcesPlaceholderConfigurer placeholderConfigurer;
    private final ClickhouseImportDefinitionsYDao clickhouseImportDefinitionsYDao;
    private final ClickhouseImportTasksYDao clickhouseImportTasksYDao;
    private final EmailSenderService emailSenderService;
    private final YtService ytService;
    private ScheduledExecutorService scheduledExecutorService;
    private Map<String, ScheduledFuture> scheduledTasks = new ConcurrentHashMap<>();
    @Value("${webmaster3.importer.locke.tasksDir}")
    private YtPath tasksDir;
    @Value("${webmaster3.importer.locke.locksNode}")
    private YtPath locksNode;
    @Value("${external.yt.service.arnold.root.default}/tmp")
    private YtPath workDirPath;

    @PostConstruct
    public void init() throws Exception {
        for (ImportDefinition definition : loadDefinitionsFromFiles()) {
            clickhouseImportDefinitionsYDao.update(definition, true);
        }

        CustomizableThreadFactory customizableThreadFactory = new CustomizableThreadFactory("importer-runner-");
        customizableThreadFactory.setDaemon(true);
        scheduledExecutorService = new ScheduledThreadPoolExecutor(WORKER_COUNT, customizableThreadFactory);
        updateDefinitions();
        ytService.inTransaction(locksNode).execute(cypressService -> {
            cypressService.create(locksNode, YtNode.NodeType.MAP_NODE, true, null, true);
            cypressService.create(YtPath.path(locksNode, ImportWithTransferManager.LOCKS_NODE), YtNode.NodeType.MAP_NODE, true, null, true);
            cypressService.create(YtPath.path(locksNode, ImportSwitchSearchQueries.LOCKS_NODE), YtNode.NodeType.MAP_NODE, true, null, true);
            return true;
        });
    }

    @PreDestroy
    public void destroy() {
        scheduledExecutorService.shutdown();
        scheduledTasks.values().forEach(scheduledFuture -> scheduledFuture.cancel(true));
        scheduledExecutorService.shutdownNow();
    }

    private List<ImportDefinition> loadDefinitionsFromFiles() throws Exception {
        PropertySourcesPropertyResolver propertyResolver = new PropertySourcesPropertyResolver(placeholderConfigurer.getAppliedPropertySources());
        List<ImportDefinition> importDefinitions = new ArrayList<>();
        var resolver = new PathMatchingResourcePatternResolver();
        for (Resource resource : resolver.getResources("/imports/*/*")) {
            if (resource.getFilename() == null) {
                continue;
            }
            try (InputStream stream = resource.getInputStream()) {
                String content = IOUtils.toString(stream, StandardCharsets.UTF_8);
                String resolvedContent = propertyResolver.resolvePlaceholders(content);
                if (resource.getFilename().endsWith(".yml")) {
                    importDefinitions.add(YAML_OM.readValue(resolvedContent, ImportDefinition.class));
                } else if (resource.getFilename().endsWith(".json")) {
                    importDefinitions.add(JsonMapping.OM.readValue(resolvedContent, ImportDefinition.class));
                }
            }
        }
        importDefinitions.forEach(ImportDefinition::validate);
        return importDefinitions;
    }

    //@Scheduled(fixedDelay = 1000 * 60 * 60)
    public void updateDefinitions() {
        Map<String, ImportTask> taskMap = clickhouseImportTasksYDao.listAll().stream().collect(Collectors.toMap(ImportTask::getId, Function.identity()));
        for (ImportDefinition definition : clickhouseImportDefinitionsYDao.listAll()) {
            ImportTask task;
            String id = definition.getId();
            if (!taskMap.containsKey(id)) {
                task = ImportTask.fromDefinition(definition);
                clickhouseImportTasksYDao.update(task);
            } else {
                task = taskMap.get(id);
            }
            if (!scheduledTasks.containsKey(id)) {
                TaskRunner taskRunner = new TaskRunner(task);
                scheduledTasks.put(id, scheduledExecutorService.schedule(taskRunner, DEFAULT_DELAY, TimeUnit.MILLISECONDS));
            }
        }
    }

    @AllArgsConstructor
    public final class TaskRunner implements Runnable {

        private ImportTask task;

        @Override
        public void run() {
            try {
                ytService.inTransaction(tasksDir).execute(cypressService -> {
                    try {
                        // refresh task
                        task = clickhouseImportTasksYDao.get(task.getId());
                        if (task.isEnabled()) {
                            processTask(cypressService);
                        }
                    } catch (Exception e) {
                        log.error("Error processing import {}", task.getId(), e);
                    }
                    return true;
                });
            } finally {
                // next execution
                scheduledTasks.put(task.getId(), scheduledExecutorService.schedule(this, getDelay(), TimeUnit.MILLISECONDS));
            }
        }

        private long getDelay() {
            if (task.getStage() == ImportStage.DONE) {
                return task.getDefinition().getDelayOnSuccess().getMillis();
            } else if (task.getStage() == ImportStage.FAILED) {
                return task.getDefinition().getDelayOnFailure().getMillis();
            } else {
                return DEFAULT_DELAY;
            }
        }

        private void processTask(YtCypressService cypressService) {
            YtPath taskPath = YtPath.path(tasksDir, task.getId());
            if (!cypressService.exists(taskPath)) {
                cypressService.create(taskPath, YtNode.NodeType.MAP_NODE, true);
            }
            try {
                cypressService.lock(taskPath, YtLockMode.EXCLUSIVE);
            } catch (YtException e) {
                log.info("Could not take lock for task {}", task.getId());
                return;
            }
            if (task.getStage().isTerminal()) {
                log.info("Starting/restarting import");
                // обновим definition
                task = task.refreshDefinition(Objects.requireNonNullElse(clickhouseImportDefinitionsYDao.get(task.getId()), task.getDefinition()));
            }
            log.info("Processing task: {}", task);
            ImportDefinition definition = task.getDefinition();

            ytService.inTransaction(definition.getInitPolicy().getCluster()).execute(workCypressService -> {
                ImportContext context = ImportContext.builder()
                        .applicationContext(applicationContext)
                        .cypressService(workCypressService)
                        .locksCypressService(cypressService)
                        .definition(definition)
                        .task(task)
                        .workDir(workDirPath)
                        .locksNode(locksNode)
                        .build();

                // processing task
                try {
                    ImportPolicy policy = definition.getImportPolicy(task.getStage());
                    if (policy == null) {
                        task = task.withNextStage().build();
                    } else {
                        task = policy.apply(context);
                    }
                } catch (Exception e) {
                    log.error("Error when processing stage {} of task {}:", task.getStage(), definition.getId(), e);
                    task = task.updateError(e.getMessage()).build();
                }
                if (task.getStage() == ImportStage.DONE && !task.getData().isEmpty()) {
                    // copy data from prevData
                    task = task.toBuilder().prevData(task.getData()).build();
                }
                // store new task state
                clickhouseImportTasksYDao.update(task);
                // store history
                if (task.getStage() == ImportStage.FAILED && !INTERRUPTED_MESSAGES.contains(task.getError())) {
                    emailSenderService.sendEmail("iceflame@yandex-team.ru", "", "Clickhouse importer " + task.getId() + " failed on stage " + task.getErrorStage(),
                            "Clickhouse importer " + task.getId() + " failed at " + task.getUpdated() + " on stage " + task.getErrorStage() + " with error: <br/>" +
                                    task.getError() + "<br/><br/>" + "Task full state: <br/>" + JsonMapping.writeValueAsPrettyString(task));
                }
                return true;
            });
        }

    }

}
