#include <maps/wikimap/mapspro/services/tasks_misc/src/import_worker/lib/dbhelpers.h>
#include <maps/wikimap/mapspro/services/tasks_misc/src/import_worker/lib/dispatch.h>
#include <maps/wikimap/mapspro/services/tasks_misc/src/import_worker/lib/gdal_helpers.h>
#include <maps/wikimap/mapspro/services/tasks_misc/src/import_worker/lib/json_helpers.h>
#include <maps/wikimap/mapspro/services/tasks_misc/src/import_worker/lib/message_reporter.h>
#include <maps/wikimap/mapspro/services/tasks_misc/src/import_worker/lib/secure_filename.h>
#include <maps/wikimap/mapspro/services/tasks_misc/src/import_worker/lib/task_params.h>

#include <maps/libs/cmdline/include/cmdline.h>
#include <maps/libs/common/include/exception.h>
#include <maps/tools/grinder/worker/include/api.h>
#include <maps/libs/concurrent/include/scoped_guard.h>
#include <maps/libs/log8/include/log8.h>
#include <maps/libs/mongo/include/init.h>
#include <yandex/maps/mds/mds.h>
#include <maps/libs/csv/include/output_stream.h>
#include <yandex/maps/wiki/common/default_config.h>
#include <yandex/maps/wiki/common/extended_xml_doc.h>
#include <yandex/maps/wiki/common/pgpool3_helpers.h>
#include <yandex/maps/wiki/common/string_utils.h>
#include <yandex/maps/wiki/tasks/task_logger.h>
#include <yandex/maps/wiki/tasks/commit_sync.h>
#include <yandex/maps/wiki/tasks/tasks.h>
#include <yandex/maps/wiki/tasks/tool_commands.h>

#include <maps/wikimap/mapspro/libs/revision_meta/include/utils.h>

#include <boost/algorithm/string/predicate.hpp>

#include <filesystem>
#include <fstream>
#include <memory>
#include <map>
#include <utility>

namespace common = maps::wiki::common;
namespace fs = std::filesystem;
namespace mlog = maps::log8;
namespace worker = maps::grinder::worker;
namespace algo = boost::algorithm;

namespace {

constexpr size_t WORKER_CONCURRENCY = 1;

} // anonymous namespace

namespace maps {
namespace wiki {
namespace importer {

namespace sql {
namespace table {
const std::string TASK = "service.task";
const std::string IMPORT_TASK = "service.import_task";
const std::string COMMIT = "service.import_commit";
const std::string MESSAGES = "service.import_messages";
}
namespace column {
const std::string ID = "id";
const std::string TASK_ID = "task_id";
const std::string COMMIT_ID = "commit_id";
const std::string DESCRIPTION = "description";
const std::string FEATURE_ID = "feature_id";
const std::string LAYER = "layer";
const std::string FILE = "file";
const std::string FILENAME = "filename";
const std::string CREATED = "created";
const std::string RESULT_URL = "result_url";
}
}

namespace {

const std::string IMPORT_XPATH = "/config/services/tasks/import";
const std::string EDITOR_CONFIG_XPATH = "/config/services/editor/config";

const std::string ARCHIVE_DIR_NAME = "input";
const std::string DATA_DIR_NAME = "data";

constexpr size_t DAYS_TO_KEEP_FILES = 30;

const std::string BYTE_ORDER_MARK = "\xef\xbb\xbf";

const std::map<std::string, Action> NAME_TO_ACTION = {
    {"add", Action::Add},
    {"edit", Action::Edit},
    {"delete", Action::Delete}
};

Action actionFromString(const std::string& value)
{
    const auto itr = NAME_TO_ACTION.find(value);
    REQUIRE(itr != NAME_TO_ACTION.end(), "Invalid action: " + value);
    return itr->second;
}

void printStatus(
    tasks::TaskPgLogger& logger,
    MessageReporter& messageReporter)
{
    if (!messageReporter.hasErrors()) {
        logger.logInfo() << "Messages count " << messageReporter.size();
        INFO() << "Messages count " << messageReporter.size();
    } else {
        logger.logError() << "Messages count " << messageReporter.size();
        ERROR() << "Messages count " << messageReporter.size();
    }
}

/// @brief Load archive file associated with the task from database to disk
fs::path loadArchiveFile(
    const fs::path& archiveDir,
    TaskParams& params)
{
    fs::create_directories(archiveDir);

    std::ostringstream query;
    query << "SELECT " << sql::column::FILE << "," << sql::column::FILENAME << std::endl
          << "FROM " << sql::table::IMPORT_TASK << std::endl
          << "WHERE " << sql::column::ID << "=" << params.taskId;

    auto txn = params.corePool.masterReadOnlyTransaction();

    auto rows = txn->exec(query.str());
    REQUIRE(!rows.empty(), "File for task " << params.taskId << " not found");
    const auto& row = rows[0];
    pqxx::binarystring bsmsg(row["file"]);
    auto filename = secureFilename(row["filename"].as<std::string>());
    auto filepath = archiveDir / filename;

    std::ofstream file;
    file.open(filepath.string(), std::ios::out | std::ios::binary);
    REQUIRE(file.is_open(), "Can't write file " << filepath.string());
    file << bsmsg.str();
    file.close();

    return filepath;
}

void decompress(
    const fs::path& archive,
    const fs::path& targetDir,
    MessageReporter& messageReporter)
{
    fs::create_directories(targetDir);

    std::ostringstream unpackCmd;
    if (archive.extension() == ".zip") {
        unpackCmd << "unzip \"" << archive.string() << "\" -d " << targetDir.string();
    } else if (algo::ends_with(archive.string(), ".tar.gz")) {
        unpackCmd << "tar xzf \"" << archive.string() << "\" -C " << targetDir.string();
    } else {
        messageReporter.error() << "Unsupported archive format " << archive.extension();
        return;
    }
    tasks::runCommandInShell(unpackCmd.str());
    INFO() << archive << " decompressed to " << targetDir;
}

void writeMessages(
    const std::vector<Message>& messages,
    TaskParams& params,
    tasks::TaskPgLogger& logger)
{
    if (messages.empty()) {
        return;
    }

    try {
        auto txn = params.corePool.masterWriteableTransaction();

        auto values = common::join(messages,
            [&](const Message& message)
            {
                return "("
                    + std::to_string(params.taskId) + ","
                    + txn->quote(message.text()) + ","
                    + (message.id() ? std::to_string(message.id()->featureId) : "NULL") + ","
                    + (message.id() ? txn->quote(message.id()->layerName) : "NULL")
                    + ")";
            },
            ',');

        std::ostringstream query;
        query << "INSERT INTO " << sql::table::MESSAGES << std::endl
            << "(" << sql::column::TASK_ID << ','
            << sql::column::DESCRIPTION << ','
            << sql::column::FEATURE_ID << ','
            << sql::column::LAYER << ")" << std::endl
            << "VALUES " << values;

        txn->exec(query.str());
        txn->commit();
    } catch (const std::exception& e) {
        ERROR() << "Message writing failed: " << e.what();
        logger.logError() << "Message writing failed: " << e.what();
    }
}

void uploadToMds(
    std::istream& stream,
    const std::string& path,
    TaskParams& params)
{
    mds::Mds mds(params.mdsConfig);
    auto response = mds.post(path, stream);

    auto url = mds.makeReadUrl(response.key());

    std::ostringstream query;
    query << "UPDATE " << sql::table::IMPORT_TASK << std::endl
        << "SET " << sql::column::RESULT_URL << "=" << params.mainTxn->quote(url) << std::endl
        << "WHERE " << sql::column::ID << "=" << params.taskId;

    auto txn = params.corePool.masterWriteableTransaction();
    txn->exec(query.str());
    txn->commit();
}

void uploadNewObjectsToMds(
    const Objects& objects,
    TaskParams& params,
    const std::string& uniqueTag)
{
    std::stringstream stream;

    /// A few hacks to make opening CSV files in Excel easier:
    /// * Add BOM at the start of the file so it can recognize UTF8.
    /// * Use ',' as decimal point separator (the default in Windows with Russian locale).
    /// * Use ';' as a CSV field separator.
    stream << BYTE_ORDER_MARK;

    csv::OutputStream csvStream(stream, ';');
    csvStream
        << "feature id"
        << "layer"
        << "db id"
        << csv::endl;

    for (const auto& object: objects) {
        csvStream
            << object->id().featureId
            << object->id().layerName
            << object->dbId()
            << csv::endl;
    }

    uploadToMds(stream, "import/" + uniqueTag + "/new_objects_" + uniqueTag + ".csv", params);
}

void uploadSkippedObjectsToMds(
    const ObjectIds& objectIds,
    TaskParams& params,
    const std::string& uniqueTag)
{
    if (objectIds.empty()) {
        return;
    }

    std::stringstream stream;
    for (auto id : objectIds) {
        stream << std::to_string(id) << "\n";
    }

    uploadToMds(stream, "import/" + uniqueTag + "/skipped_" + uniqueTag + ".txt", params);
}

CommitIds doImport(
    const worker::Task& task,
    TaskParams& params,
    const std::string& uniqueTag,
    bool dryRun,
    tasks::TaskPgLogger& logger)
{
    MessageReporter messageReporter;
    concurrent::ScopedGuard guard([&]()
    {
        printStatus(logger, messageReporter);
        writeMessages(messageReporter.messages(), params, logger);
    });

    auto archiveDir = params.workDir / ARCHIVE_DIR_NAME;
    auto dataDir = params.workDir / DATA_DIR_NAME;
    fs::path archivePath;

    runLoggable([&]{ archivePath = loadArchiveFile(archiveDir, params); },
            logger, "Load archive file");

    logger.logInfo() << "Archive file: " << archivePath;
    INFO() << "Archive file: " << archivePath;

    runLoggable([&]{ decompress(archivePath, dataDir, messageReporter); },
            logger, "Decompress archive file");

    if (messageReporter.hasErrors()) {
        return {};
    }

    ObjectsCache cache;
    Objects objects;

    runLoggable([&]
        {
            objects = gdal2objects(
                dataDir,
                params.action,
                cache,
                params.editorConfig,
                params.importConfig,
                messageReporter);
        },
        logger, "Convert gdal to objects");

    INFO() << "Object count " << objects.size();
    logger.logInfo() << "Object count " << objects.size();
    printStatus(logger, messageReporter);

    if (messageReporter.hasErrors() || objects.empty()) {
        return {};
    }

    if (task.isCanceled()) {
        throw worker::TaskCanceledException{};
    }

    DispatchResult result;

    runLoggable([&]
        {
            result = dispatchObjects(cache, params, logger, messageReporter);
        },
        logger, "Dispatch objects");

    if (messageReporter.hasErrors()) {
        return {};
    }

    if (params.action == Action::Add) {
        ASSERT(!result.commitIds.empty());
        ASSERT(!objects.empty());
        runLoggable([&]{ uploadNewObjectsToMds(objects, params, uniqueTag); },
            logger, "Upload new objects to MDS");
    } else {
        runLoggable([&]{ uploadSkippedObjectsToMds(result.skippedObjectIds, params, uniqueTag); },
            logger, "Upload skipped objects to MDS");
    }

    if (!result.commitIds.empty()) {
        tasks::CommitRecordGateway commitGtw(params.mainTxn, sql::table::COMMIT);
        commitGtw.addCommits(params.taskId, result.commitIds, tasks::CommitStatus::NotSynced);
    }

    if (task.isCanceled()) {
        throw worker::TaskCanceledException{};
    }

    if (!dryRun) {
        params.mainTxn->commit();
    }

    return result.commitIds;
}

void refreshViews(
    const CommitIds& commitIds,
    TaskParams& params,
    const std::string& configPath,
    tasks::TaskPgLogger& logger)
{
    try {
        runLoggable([&]
        {
            tasks::refreshViews(commitIds, params.branchId, configPath);
        },
        logger, "Refresh view");

        auto txn = params.corePool.masterWriteableTransaction();
        tasks::CommitRecordGateway commitGtw(txn, sql::table::COMMIT);
        commitGtw.updateStatus(params.taskId, commitIds, tasks::CommitStatus::Synced);
        txn->commit();

    } catch (...) {
        logger.logError() << "Refreshing view failed";
        auto txn = params.corePool.masterWriteableTransaction();
        tasks::freezeTask(*txn, params.taskId);
        txn->commit();

        throw;
    }
}

void deleteOldFilesFromTable(TaskParams& params)
{
    auto txn = params.corePool.masterWriteableTransaction();

    std::ostringstream query;
    query << "UPDATE " << sql::table::IMPORT_TASK << std::endl
          << "SET " << sql::column::FILE << "=NULL" << std::endl
          << "WHERE " << sql::column::ID << " IN " << std::endl
                << "(SELECT " << sql::column::ID << std::endl
                << "FROM " << sql::table::IMPORT_TASK << " INNER JOIN " << sql::table::TASK << std::endl
                << "USING (" << sql::column::ID << ")" << std::endl
                << "WHERE " << sql::column::CREATED << " < current_date - " << DAYS_TO_KEEP_FILES << std::endl
                << "AND " << sql::column::FILE << " IS NOT NULL)";

    txn->exec(query.str());
    txn->commit();
}

} // namespace

void importWorker(const worker::Task& task,
                  const common::ExtendedXmlDoc& configXml)
{
    const auto& args = task.args();

    common::PoolHolder dbHolder(configXml, "core", "grinder");
    common::PoolHolder viewDbHolder(configXml, "view-trunk", "editor-tool");

    auto configPath = configXml.sourcePaths()[0];
    auto taskId = args["taskId"].as<uint64_t>();
    auto uniqueTag = tasks::makeTaskTag(taskId);
    bool dryRun = args["dryRun"].as<bool>();

    fs::path baseDir = configXml.getAttr<std::string>(IMPORT_XPATH, "workDir");
    fs::path workDir = baseDir / uniqueTag;
    fs::create_directories(workDir);

    TaskParams params(
        taskId,
        args["uid"].as<uint64_t>(),
        actionFromString(args["action"].as<std::string>()),
        configXml,
        configXml.get<std::string>(EDITOR_CONFIG_XPATH),
        workDir,
        dbHolder.pool(),
        viewDbHolder.pool());

    tasks::TaskPgLogger logger(params.corePool, params.taskId);
    logger.logInfo() << "Task started. Grinder task id: " << task.id();

    if (dryRun) {
        logger.logWarn() << "Dry run mode is enabled";
    }

    try {
        // If the task is 'frozen', it will resume at view refresh stage
        bool frozen = tasks::isFrozen(*params.mainTxn, params.taskId);
        logger.logInfo() << "Task " << (frozen ? "resumed" : "started");

        CommitIds commits;

        if (frozen) {
            tasks::CommitRecordGateway commitGtw(params.mainTxn, sql::table::COMMIT);
            commits = commitGtw.getCommitIds<CommitIds>(params.taskId);
            params.mainTxn.releaseConnection(); //do not hold open transaction during refresh stage
        } else {
            commits = doImport(task, params, uniqueTag, dryRun, logger);
            params.mainTxn.releaseConnection(); //do not hold open transaction during refresh stage

            if (!commits.empty()) {
                if (!dryRun) {
                    runLoggable([&] {
                            auto txn = params.corePool.masterWriteableTransaction();
                            revision_meta::enqueueServiceTaskCommitsForApprovedBranch(
                                *txn,
                                revision_meta::TCommitIds{commits.cbegin(), commits.cend()}
                            );
                            txn->commit();
                        },
                        logger, "Enqueue commits for approved branch");
                }
            } else {
                logger.logError() << "Objects were not imported!";
            }
        }

        if (!commits.empty() && !dryRun) {
            refreshViews(commits, params, configPath, logger);
        }

        if (frozen) {
            auto txn = params.corePool.masterWriteableTransaction();
            tasks::unfreezeTask(*txn, params.taskId);
            txn->commit();
        }

        deleteOldFilesFromTable(params);
        tasks::deleteOldTmpFiles(baseDir.string());
    } catch (const worker::TaskCanceledException&) {
        WARN() << "Import canceled";
        logger.logWarn() << "Task canceled";
        throw;
    } catch (const maps::Exception& e) {
        ERROR() << "Import failed: " << e;
        logger.logError() << "Task failed: " << e.what();
        throw;
    } catch (const std::exception& e) {
        ERROR() << "Import failed: " << e.what();
        logger.logError() << "Task failed: " << e.what();
        throw;
    }

    logger.logInfo() << "Task finished";
}

} // importer
} // wiki
} // maps

int main(int argc, char* argv[]) try
{
    maps::cmdline::Parser parser;
    auto workerConfig = parser.string("config")
        .help("path to worker configuration");
    auto syslogTag = parser.string("syslog-tag")
        .help("redirect log output to syslog with given tag");
    auto grinderConfig = parser.string("grinder-config")
        .help("path to grinder configuration file");

    parser.parse(argc, argv);

    if (syslogTag.defined()) {
        mlog::setBackend(mlog::toSyslog(syslogTag));
    }

    maps::mongo::init();

    std::unique_ptr<common::ExtendedXmlDoc> configDocPtr;
    if (workerConfig.defined()) {
        configDocPtr.reset(new common::ExtendedXmlDoc(workerConfig));
    } else {
        configDocPtr = common::loadDefaultConfig();
    }

    worker::Options workerOpts;

    if (grinderConfig.defined()) {
        workerOpts.setConfigLocation(grinderConfig);
    }

    workerOpts.on("import",
            [&](const worker::Task& task) {
                maps::wiki::importer::importWorker(task, *configDocPtr);
            })
        .setConcurrencyLevel(WORKER_CONCURRENCY);

    worker::run(std::move(workerOpts));

    return EXIT_SUCCESS;
} catch (const maps::Exception& e) {
    ERROR() << "Worker failed: " << e;
    return EXIT_FAILURE;
} catch (const std::exception& e) {
    ERROR() << "Worker failed: " << e.what();
    return EXIT_FAILURE;
}
