#include "revision2json.h"
#include "pg_helpers.h"
#include "helpers.h"
#include "json2ymapsdf.h"
#include "measurable_task.h"

#include <yandex/maps/wiki/common/retry_duration.h>
#include <yandex/maps/wiki/revision/revisionsgateway.h>
#include <yandex/maps/wiki/revisionapi/revisionapi.h>
#include <yandex/maps/wiki/threadutils/executor.h>

#include <maps/libs/log8/include/log8.h>

#include <boost/format.hpp>

#include <fstream>
#include <iomanip>
#include <sstream>
#include <mutex>

namespace maps::wiki::exporter {

namespace {

const size_t BATCH_LIMIT = 10000;
const size_t TASK_LIMIT = 1000000;
const size_t SUBTASK_LIMIT = TASK_LIMIT / 10;
const std::string FILE_NAME_SUFFIX = "-%d-%%|03d|.json"; //%% will be formatted on second step


template <class Duration>
typename Duration::rep toMillisec(const Duration& dur)
{
    using namespace std::chrono;
    return duration_cast<milliseconds>(dur).count();
}


rev::DBID lastObjectId(pgpool3::Pool& pool)
{
    return common::retryDuration([&] {
        auto txn = pool.masterReadOnlyTransaction();
        revision::RevisionsGateway gtw(*txn);
        return gtw.lastObjectId();
    });
}


class logDeleter
{
public:
    logDeleter(std::ostream& log, std::string message)
        : log_(log)
        , message_(std::move(message))
    {}

    void operator()(std::ofstream* ptr)
    {
        static std::mutex mutex;
        std::lock_guard<std::mutex> lock(mutex);
        std::default_delete<std::ofstream>()(ptr);
        log_ << message_;
    }

private:
    std::ostream& log_;
    std::string message_;
};


class LoggingChunkStreamWrapper
{
public:
    typedef std::shared_ptr<std::ostream> return_type;

    LoggingChunkStreamWrapper(std::ostream& log, std::string filePathPattern)
        : log_(log)
        , filePathPattern_(std::move(filePathPattern))
    { }

    return_type operator()(size_t chunkNo) const
    {
        const auto filePath = (boost::format(filePathPattern_) % chunkNo).str();
        const auto deleter = logDeleter(log_, filePath + "\n");
        return_type stream(new std::ofstream(filePath), deleter);
        REQUIRE(!stream->fail(), "Error opening '" + filePath + "' for writing");
        return stream;
    }

private:
    std::ostream& log_;
    std::string filePathPattern_;
};

} // namespace

void exportJsons(
    const ExportConfig& exportCfg,
    const revisionapi::ExportParams& params,
    const revisionapi::GetStreamForChunkFunc& streamCallback,
    const revisionapi::FilterPtr& filter)
{
    std::map<size_t, std::shared_ptr<std::stringstream>> streams;

    auto memoryWriter = [&](size_t index) -> std::shared_ptr<std::ostream> {
        auto newStream = std::make_shared<std::stringstream>();
        streams[index] = newStream;
        return newStream;
    };

    auto& pool = exportCfg.mainPool();

    common::retryDuration([&] {
        streams.clear();
        try {
            revisionapi::RevisionAPI api(pool);
            api.exportData(params, memoryWriter, filter);
        } catch (const revisionapi::DataError& ex) {
            throw common::RetryDurationCancel() << ex.what();
        }
    });

    for (auto& [index, streamPtr] : streams) {
        auto newStreamPtr = streamCallback(index);
        *newStreamPtr << streamPtr->rdbuf();
    }
}

/// Export json for domain/masstransit categories
void revision2Json(
    const ExportConfig& exportCfg,
    const ExportFiles& exportFiles)
{
    using Clock = std::chrono::high_resolution_clock;
    auto start = Clock::now();

    JsonPass commonPass;
    commonPass.name = "common";
    commonPass.commitId = exportCfg.commitId();

    fs::path outputDirPath = fs::absolute(fs::path(exportFiles(TmpFile::JSON_DIR)));

    auto& pool = exportCfg.mainPool();
    auto branch = loadBranchByString(pool, exportCfg.branch());
    auto commitId = exportCfg.commitId();
    REQUIRE(commitId, "export from empty database");
    auto params = revisionapi::ExportParams::loadFromFile(
        branch, commitId, exportFiles(TmpFile::PRINTED_CFG));
    params->setWriteBatchSize(BATCH_LIMIT);
    params->setEmptyJsonPolicy(revisionapi::EmptyJsonPolicy::Skip);

    std::ofstream jsonList(exportFiles(TmpFile::JSON_LIST_FILE));
    std::vector<MeasurableTask> tasks;
    revision::DBID id = 0;
    const auto lastId = lastObjectId(pool);
    while (id <= lastId) {
        const auto endId = std::min(lastId, id + TASK_LIMIT - 1);

        std::ostringstream label;
        label << "json pass " << id / TASK_LIMIT + 1 << "/" << lastId / TASK_LIMIT + 1;

        std::ostringstream fileNamePattern;
        fileNamePattern << std::setw(6) << std::setfill('0') << id / TASK_LIMIT << FILE_NAME_SUFFIX;
        const auto filePathPattern = outputDirPath.string() + "/" + fileNamePattern.str();

        tasks.emplace_back([&exportCfg, &params, &jsonList, filePathPattern, id, endId] {
            size_t subtaskId = 0;

            for (auto subtaskStartId = id;
                 subtaskStartId <= endId;
                 subtaskStartId += SUBTASK_LIMIT, ++subtaskId)
            {
                exportCfg.checkCanceled();

                const auto preformattedFilePath = (boost::format(filePathPattern) % subtaskId).str();
                LoggingChunkStreamWrapper streamCallback(jsonList, preformattedFilePath);

                const auto subtaskEndId = std::min(endId, subtaskStartId + SUBTASK_LIMIT - 1);
                revision::DBIDSet objectIdBetweenFilter = {subtaskStartId, subtaskEndId};
                revisionapi::FilterPtr filter =
                    std::make_shared<rev::filters::TableAttrFilterExpr>(
                        revision::filters::ObjRevAttr::objectId().between(
                            objectIdBetweenFilter));

                exportJsons(exportCfg, *params, streamCallback, filter);
            }
        }, exportCfg.logger(), label.str(), LogTaskMode::OnFinish);

        id = endId + 1;
    }

    Executor executor;
    for (size_t i = 0; i < tasks.size(); ++i) {
        executor.addTask([&tasks, i]{tasks[i]();});
    }

    const auto slaveMaxSize =
        exportCfg.mainPool().state().constants.slaveMaxSize;
    INFO() << "Working threads size: " << slaveMaxSize;
    ThreadPool threadPool(slaveMaxSize);
    executor.executeAllInThreads(threadPool);

    Clock::duration duration = Clock::now() - start;
    Clock::duration passesSum(std::chrono::milliseconds(0));

    std::ostringstream stats;
    for (const auto& task: tasks) {
        passesSum += task.duration();
    }
    stats << "all passes duration: " << FormattedDuration(duration)
          << "\npasses sum: " << FormattedDuration(passesSum)
          << "\nspeedup: " << std::fixed << std::setprecision(3)
          << double(toMillisec(passesSum)) / toMillisec(duration);

    INFO() << stats.str();
    exportCfg.logger().logInfo() << stats.str();
}

} // namespace maps::wiki::exporter

