#include "multipart_builder.h"

#include <maps/wikimap/mapspro/services/mrc/long_tasks/async_takeout_uploader/lib/takeout.h>
#include <maps/wikimap/mapspro/services/mrc/libs/db/include/feature_gateway.h>
#include <maps/wikimap/mapspro/services/mrc/libs/db/include/track_point_gateway.h>

#include <maps/libs/chrono/include/time_point.h>
#include <maps/libs/common/include/environment.h>
#include <maps/libs/json/include/builder.h>
#include <maps/libs/sql_chemistry/include/batch_load.h>
#include <maps/libs/common/include/make_batches.h>
#include <maps/libs/http/include/http.h>
#include <maps/libs/log8/include/log8.h>
#include <maps/wikimap/mapspro/services/mrc/libs/common/include/algorithm/retry.h>
#include <yandex/maps/wiki/common/pgpool3_helpers.h>
#include <yandex/maps/wiki/editor_takeout/editor_takeout.h>

#include <algorithm>
#include <memory>
#include <optional>
#include <type_traits>

namespace maps::mrc::takeout {
namespace {

using Strings = std::vector<std::string>;

// Assuming that photos are taken withing one minute after an user taps a
// recording button in the application.
constexpr auto RIDE_DURATION_MARGIN = std::chrono::minutes{1};
constexpr auto RIDE_BRAKE = std::chrono::minutes{5};
constexpr std::size_t DB_BATCH_SIZE = 4000;
constexpr std::size_t UPLOAD_BATCH_SIZE = 10000;

const std::string NMAPS_PREFIX = "nmaps_";
const std::string TRACKS_FILENAME_PREFIX = NMAPS_PREFIX + "tracks_";
const std::string IMAGES_FILENAME_PREFIX = NMAPS_PREFIX + "images_";
const TString TAKEOUT_SERVICE_ALIAS = "takeout";

struct RideSpec {
    chrono::TimePoint begin;
    chrono::TimePoint end;
    std::string sourceId;

    void addMargins()
    {
        begin -= RIDE_DURATION_MARGIN;
        end += RIDE_DURATION_MARGIN;
    }
};

std::vector<RideSpec> makeRideSpecs(const db::Features& features)
{
    std::vector<RideSpec> rideSpecs;
    if (features.empty()) {
        return rideSpecs;
    }

    rideSpecs.push_back(RideSpec{features.front().timestamp(),
                                 features.front().timestamp(),
                                 features.front().sourceId()});

    for (const auto& feature : features) {
        if (RIDE_BRAKE < feature.timestamp() - rideSpecs.back().end
            || feature.sourceId() != rideSpecs.back().sourceId) {

            rideSpecs.back().addMargins();
            rideSpecs.push_back(RideSpec{feature.timestamp(),
                                         feature.timestamp(),
                                         feature.sourceId()});
            continue;
        }
        rideSpecs.back().end = feature.timestamp();
    }
    rideSpecs.back().addMargins();
    return rideSpecs;
}

db::Features loadPhotos(pgpool3::Pool& pool, const std::string& uid)
{
    db::Features features;

    auto txn = pool.slaveTransaction();
    sql_chemistry::BatchLoad<db::table::Feature> batch{
        DB_BATCH_SIZE,
        db::table::Feature::userId == uid &&
            !db::table::Feature::gdprDeleted.is(true)};

    while (batch.next(*txn)) {
        features.insert(features.end(), batch.begin(), batch.end());
    }

    return features;
}

db::TrackPoints loadRideTracks(pgpool3::Pool& pool,
                               const std::vector<RideSpec>& rideSpecs)
{
    db::TrackPoints trackPoints;
    if (rideSpecs.empty()) {
        return trackPoints;
    }

    auto txn = pool.slaveTransaction();

    for (const auto& rideSpec : rideSpecs) {
        auto rideTrackPoints = db::TrackPointGateway{*txn}.load(
            db::table::TrackPoint::sourceId == rideSpec.sourceId &&
            db::table::TrackPoint::timestamp.between(rideSpec.begin,
                                                     rideSpec.end));
        trackPoints.insert(trackPoints.end(),
                           std::make_move_iterator(rideTrackPoints.begin()),
                           std::make_move_iterator(rideTrackPoints.end()));
    }

    return trackPoints;
}

std::string makeImageUrl(const db::Feature& feature)
{
    return std::string{"https://mrc-browser.maps.yandex.ru/feature/"}
           + std::to_string(feature.id()) + "/image";
}

} // anonymous namespace

TakeoutUploader::TakeoutUploader(
    const common::Config& config,
    const wiki::common::ExtendedXmlDoc& nmapsConfig)
    : takeoutConfig_{config.externals().takeoutConfig()}
    , poolHolder_{config.makePoolHolder(maps::mrc::common::LONG_READ_DB_ID,
                                        maps::mrc::common::LONG_READ_POOL_ID)}
    , nmapsLongReadPool_(nmapsConfig, "long-read", "long-read")
    , nmapsSocialPool_(nmapsConfig, "social", "grinder")
{
    const char* TVM_CACHE_DIRECTORY
        = "/var/cache/yandex/maps/mrc/async-takeout-uploader/tvm";

    class TvmLogger : public NTvmAuth::ILogger {
    public:
        void Log(int lvl, const TString& msg) override
        {
            MAPS_LOG(static_cast<log8::Level>(lvl)) << "tvm ticket parser: "
                                                    << msg;
        }
    };

    auto env = maps::common::getYandexEnvironment();
    if (env == maps::common::Environment::Testing
        || env == maps::common::Environment::Datatesting
        || env == maps::common::Environment::Stable) {
        NTvmAuth::TLoggerPtr logger = MakeIntrusive<TvmLogger>();
        NTvmAuth::NTvmApi::TClientSettings settings;
        settings.SetSelfTvmId(takeoutConfig_.ownTvmServiceId());
        settings.EnableServiceTicketChecking();
        settings.EnableServiceTicketsFetchOptions(
            TString(takeoutConfig_.tvmSecret()),
            {{TAKEOUT_SERVICE_ALIAS, takeoutConfig_.dstTvmServiceId()}});
        settings.SetDiskCacheDir(TVM_CACHE_DIRECTORY);
        tvmClient_.emplace(settings, logger);
    }
}

void TakeoutUploader::handleTakeoutUpload(const std::string& uid,
                                          const std::string& takeoutJobId)
{
    common::retryOnException<std::exception>(
        common::RetryPolicy()
            .setInitialTimeout(std::chrono::seconds(60))
            .setMaxAttempts(12)
            .setTimeoutBackoff(2),
        [&]{ handleTakeoutUploadImpl(uid, takeoutJobId); });
}

void TakeoutUploader::handleTakeoutUploadImpl(const std::string& uid,
                                              const std::string& takeoutJobId)
{
    INFO() << "Exporting user data for UID " << uid
           << ", takeout jobId: " << takeoutJobId;

    // Avoid races. Takeout may be not ready to receive data immediately.
    std::this_thread::sleep_for(std::chrono::seconds{3});

    // User ID can be found in the ride/walk photos only. So, load all ride
    // photos for a given UID, split them into rides, and then load track
    // points using the collected source_ids and border timestamps (with
    // some margins).
    auto features = loadPhotos(poolHolder_.pool(), uid);

    // Sort user features to split them into rides
    std::sort(features.begin(), features.end(),
              [](const auto& lhs, const auto& rhs) {
                  return std::make_tuple(lhs.sourceId(), lhs.timestamp())
                       < std::make_tuple(rhs.sourceId(), rhs.timestamp());
              });
    const auto rideSpecs = makeRideSpecs(features);

    const auto timestampsCmp = [](const auto& lhs, const auto& rhs) {
        return lhs.timestamp() < rhs.timestamp();
    };
    std::sort(features.begin(), features.end(), timestampsCmp);

    http::Client httpClient;
    httpClient.setTimeout(std::chrono::seconds{60});
    auto imgFilenames = uploadImagesJson(httpClient, takeoutJobId, features);

    auto trackPoints = loadRideTracks(poolHolder_.pool(), rideSpecs);
    std::sort(trackPoints.begin(), trackPoints.end(), timestampsCmp);
    auto trackFilenames
        = uploadTracksJson(httpClient, takeoutJobId, trackPoints);

    auto editorFilenames = uploadNmapsData(httpClient, uid, takeoutJobId);

    Strings allFilenames;
    for (auto fnames : {&imgFilenames, &trackFilenames, &editorFilenames}) {
        std::move(fnames->begin(), fnames->end(), std::back_inserter(allFilenames));
    }

    notifyUploadingDone(httpClient, takeoutJobId, allFilenames);
    INFO() << "Done with nmaps takeout export for UID " << uid;
}

void TakeoutUploader::addTvmTicketIfNeeded(http::Request& request)
{
    auto env = maps::common::getYandexEnvironment();
    if (env == maps::common::Environment::Testing
        || env == maps::common::Environment::Datatesting
        || env == maps::common::Environment::Stable
    ) {
        request.addHeader(
            "X-Ya-Service-Ticket",
            tvmClient_->GetServiceTicketFor(TAKEOUT_SERVICE_ALIAS)
        );
    }
}

void TakeoutUploader::performHttpRequestWithRetry(
    HttpRequestProvider requestProvider)
{
    common::retryOnException<http::Error>(
        common::RetryPolicy()
            .setInitialTimeout(std::chrono::seconds(5))
            .setMaxAttempts(15)
            .setTimeoutBackoff(2),
        [&]() {
            auto request = requestProvider();

            // Refresh TVM ticket each time
            addTvmTicketIfNeeded(request);

            try {
                auto response = request.perform();

                // https://wiki.yandex-team.ru/passport/takeout/integration
                // For request to succeed:
                //   1. HTTP status must be 200,
                //   2. 'status' in response body must be 'ok'.
                // Otherwise just retry.
                const auto body = response.readBody();
                const auto status = json::Value::fromString(body)["status"].as<std::string>();

                if (response.status() != 200 || status != "ok") {
                    http::Error error;
                    error << response.url().toString()
                        << ", status: " << response.status()
                        << ", body: " << body;
                    ERROR() << "http request failed. status: " << response.status()
                            << ", body: " << body;
                    throw error;
                }
            } catch (const maps::Exception& ex) {
                ERROR() << "Exception till performing request "
                    << request.url() << ": " << ex;
                throw;
            } catch (const std::exception& ex) {
                ERROR() << "Exception till performing request "
                    << request.url() << ": " << FormatCurrentException();
                throw;
            }

        });
}

void TakeoutUploader::uploadToTakeout(http::Client& httpClient,
                                      const std::string& takeoutJobId,
                                      const std::string& filename,
                                      const std::string& json)
{
    auto takeoutUrl = takeoutConfig_.url();
    takeoutUrl.setPath("/1/upload/");

    auto requestProvider = [&]() {
        http::Request request{httpClient, http::POST, takeoutUrl};
        request.addParam("consumer", "mrc");

        MultipartBuilder multipartBuilder;
        multipartBuilder.add(NameValueFormData{"job_id", takeoutJobId});
        multipartBuilder.add(JsonAttachmentFormData{filename, json});
        multipartBuilder.fill(request);
        return request;
    };

    INFO() << "Uploading file " << filename;

    performHttpRequestWithRetry(requestProvider);
}

void TakeoutUploader::notifyUploadingDone(
    http::Client& httpClient,
    const std::string& takeoutJobId,
    const Strings& filenames)
{
    auto takeoutUrl = takeoutConfig_.url();
    takeoutUrl.setPath("/1/upload/done/");

    auto requestProvider = [&]() {
        http::Request request{httpClient, http::POST, takeoutUrl};
        request.addParam("consumer", "mrc");

        MultipartBuilder multipartBuilder;
        multipartBuilder.add(NameValueFormData{"job_id", takeoutJobId});

        const auto addFilenamesToForm =
            [&](const Strings& filenames) {
                for (const auto& filename : filenames) {
                    multipartBuilder.add(NameValueFormData{"filename", filename});
                }
            };
        addFilenamesToForm(filenames);

        multipartBuilder.fill(request);
        return request;
    };

    INFO() << "Notify upload done";
    performHttpRequestWithRetry(requestProvider);
}

Strings
TakeoutUploader::uploadImagesJson(http::Client& httpClient,
                                  const std::string& takeoutJobId,
                                  const db::Features& features)
{
    // [
    //     {
    //       "date" : "2018-03-14 17:37:19.132",
    //       "lat" : 55.821335,
    //       "lon" : 37.573273,
    //       "url" :
    //       "https://mrc-browser.maps.yandex.ru/feature/15723965/image"
    //     },
    //     ...
    // ]

    std::size_t count = 0;
    Strings filenames;
    for (const auto& batch :
         maps::common::makeBatches(features, UPLOAD_BATCH_SIZE)) {

        maps::json::Builder builder;
        builder.setDoublePrecision(8);
        builder << [&](maps::json::ArrayBuilder builder) {
            for (const auto& feature : batch) {
                builder << [&](maps::json::ObjectBuilder builder) {
                    builder["date"]
                        << chrono::formatIsoDateTime(feature.timestamp());
                    if (feature.hasPos()) {
                        const auto pos = feature.geodeticPos();
                        builder["lat"] << pos.y();
                        builder["lon"] << pos.x();
                    }
                    builder["url"] << makeImageUrl(feature);
                };
            }
        };

        auto filename
            = IMAGES_FILENAME_PREFIX + std::to_string(++count) + ".json";
        uploadToTakeout(httpClient, takeoutJobId, filename, builder.str());
        filenames.push_back(filename);
    }
    return filenames;
}

Strings
TakeoutUploader::uploadTracksJson(http::Client& httpClient,
                                  const std::string& takeoutJobId,
                                  const db::TrackPoints& trackPoints)
{
    // [
    //   {
    //     "date" : "2018-03-14 17:34:19.064",
    //     "lat" : 55.821335,
    //     "lon" : 37.573273
    //   },
    //   ...
    // ]

    Strings filenames;
    std::size_t count = 0;
    for (const auto& batch :
         maps::common::makeBatches(trackPoints, UPLOAD_BATCH_SIZE)) {
        maps::json::Builder builder;
        builder.setDoublePrecision(8);
        builder << [&](maps::json::ArrayBuilder builder) {
            for (const auto& trackPoint : batch) {
                builder << [&](maps::json::ObjectBuilder builder) {
                    builder["date"]
                        << chrono::formatIsoDateTime(trackPoint.timestamp());
                    const auto pos = trackPoint.geodeticPos();
                    builder["lat"] << pos.y();
                    builder["lon"] << pos.x();
                };
            }
        };

        auto filename
            = TRACKS_FILENAME_PREFIX + std::to_string(++count) + ".json";
        uploadToTakeout(httpClient, takeoutJobId, filename, builder.str());
        filenames.push_back(filename);
    }
    return filenames;
}

Strings TakeoutUploader::uploadNmapsData(http::Client& httpClient,
                                         const std::string& uid,
                                         const std::string& takeoutJobId)
{
    Strings filenames;
    wiki::editor_takeout::UploadToTakeout editorDataUploader
        = [&](const std::string& filenameSuffix, const std::string& data) {
            std::string filename = NMAPS_PREFIX + filenameSuffix;
            uploadToTakeout(httpClient, takeoutJobId, filename, data);
            filenames.push_back(std::move(filename));
        };

    uint64_t intUid = std::stoull(uid);
    wiki::editor_takeout::generateTakeoutEdits(
        nmapsLongReadPool_.pool(), intUid, editorDataUploader);

    wiki::editor_takeout::generateTakeoutMessages(
        nmapsSocialPool_.pool(), intUid, editorDataUploader);

    return filenames;
}

} // namespace maps::mrc::takeout
