#include "loader.h"
#include "utils.h"

#include <maps/wikimap/mapspro/libs/gdpr/include/user.h>
#include <maps/wikimap/mapspro/libs/poi_feed/include/helpers.h>
#include <maps/wikimap/mapspro/libs/poi_feed/include/magic_strings.h>
#include <maps/wikimap/mapspro/libs/poi_feed/include/reserved_id.h>

#include <maps/libs/log8/include/log8.h>
#include <yandex/maps/wiki/common/batch.h>
#include <yandex/maps/wiki/common/string_utils.h>
#include <yandex/maps/wiki/revision/commit.h>
#include <yandex/maps/wiki/revision/filters.h>
#include <yandex/maps/wiki/common/retry_duration.h>

#include <algorithm>
#include <chrono>
#include <unordered_map>

#include <boost/algorithm/string.hpp>
#include <boost/lexical_cast.hpp>

using namespace std::string_literals;

namespace maps::wiki::poi {
namespace {

const auto PROTECTED_FT_TYPES_RESOURCE = "PROTECTED_FT_TYPES_RESOURCE"s;

enum class NameType {
    Official = 0,
    RenderLabel = 1,
    Synonym = 4,
};

const std::unordered_map<std::string, std::string> openTypeToPublishingStatus
{
    {"1", "publish"},
    {"2", "obsolete"},
    {"3", "temporarily_closed"},
    {"4", "temporarily_closed"},
    {"5", "closed"}
};

const std::string POSITION_QUALITY_FT_ATTR = "position_quality";

const std::string REL_ROLE_ATTR = "rel:role";
const std::string ROLE_ENTRANCE_ASSIGNED = "entrance_assigned";

const std::string MULTIVALUE_DELIMITER = "|";

namespace tbl {
const auto FT = "ft"s;
const auto FT_ATTR = "ft_attr"s;
const auto FT_CENTER = "ft_center"s;
const auto FT_NM = "ft_nm"s;
const auto NODE = "node"s;
const auto ENTRANCES = "entrances"s;
} // namespace tbl

namespace col {
const auto DISP_CLASS = "disp_class"s;
const auto FT_ID = "ft_id"s;
const auto POI_FT_ID = "poi_ft_id"s;
const auto ENTRANCE_ID = "entrance_id"s;
const auto FT_TYPE_ID = "ft_type_id"s;
const auto POI_FT_TYPE_ID = "poi_ft_type_id"s;
const auto PARENT_FT_TYPE_ID = "parent_ft_type_id"s;
const auto KEY = "key"s;
const auto LANG = "lang"s;
const auto NAME = "name"s;
const auto NAME_TYPE = "name_type"s;
const auto NM_ID = "nm_id"s;
const auto NODE_ID = "node_id"s;
const auto SHAPE = "shape"s;
const auto VALUE = "value"s;
} // namespace col

namespace alias {
const auto X = "x"s;
const auto Y = "y"s;
} // namespace alias

DBIDSet getPoiIds(const PoiIdToDefMap& poiIdToDefMap)
{
    DBIDSet result;
    for (const auto& poiIdToDef : poiIdToDefMap) {
        result.insert(poiIdToDef.first);
    }
    return result;
}

DBIDSet getPoiWithoutAccurateRubricId(const PoiIdToDefMap& poiIdToDefMap)
{
    DBIDSet result;
    for (const auto& poiIdToDef : poiIdToDefMap) {
        if (!poiIdToDef.second.userDefinedRubricId) {
            result.insert(poiIdToDef.first);
        }
    }
    return result;
}

std::string publishingStatus(const std::string& openType, DBID objectId)
{
    auto it = openTypeToPublishingStatus.find(openType);
    if (it == openTypeToPublishingStatus.end()) {
        WARN() << "Object :" << objectId << " invalid openType:" << openType;
        return {};
    }
    return it->second;
}

} // anonymous namespace

Loader::Loader(pqxx::transaction_base& yMapsDf,
           pgpool3::Pool& socialPool,
           revision::Snapshot& tds,
           const FtTypeIdToRubricIdMultimap& mapping,
           const FtTypeIdToRubricIdMultimap& parentMapping,
           ObjectIdToPermalinkUnknownMap&& nyakMappingUnknownPermalinks)
        : yMapsDf_(yMapsDf)
        , socialPool_(socialPool)
        , tds_(tds)
        , mapping_(mapping)
        , parentMapping_(parentMapping)
        , nyakMappingUnknownPermalinks_(std::move(nyakMappingUnknownPermalinks))
{
}

FtTypeIdSet Loader::getFtTypeIds() const
{
    FtTypeIdSet result;
    for (const auto& pair : mapping_) {
        result.insert(pair.first);
    }
    return result;
}

DBIDSet Loader::loadPoiIds()
{
    DBIDSet result;
    std::stringstream query;
    query << "SELECT " << tbl::FT << "." << col::FT_ID << " FROM " << tbl::FT
          << " JOIN " << tbl::FT_CENTER << " USING (" << col::FT_ID
          << ") WHERE " << col::FT_TYPE_ID << " IN ("
          << wiki::common::join(getFtTypeIds(), ",")
          << ")";
    auto rows = yMapsDf_.exec(query.str());

    for (const auto& row : rows) {
        result.insert(row[col::FT_ID].as<DBID>());
    }
    return result;
}

void Loader::loadFtData(PoiIdToDefMap& poiIdToDefMap)
{
    std::stringstream query;
    query << "SELECT " << col::FT_ID << ", " << col::DISP_CLASS
          << ", " << col::FT_TYPE_ID
          << " FROM " << tbl::FT << " WHERE " << col::FT_ID << " IN ("
          << wiki::common::join(getPoiIds(poiIdToDefMap), ",") << ")";
    auto rows = yMapsDf_.exec(query.str());

    REQUIRE(rows.size() == poiIdToDefMap.size(),
        "Some poi are missing from FT table!");
    for (const auto& row : rows) {
        const auto poiId = row[col::FT_ID].as<DBID>();
        auto& def = poiIdToDefMap[poiId];
        def.dispClass = row[col::DISP_CLASS].as<DispClass>();
        auto ftTypeId = row[col::FT_TYPE_ID].as<FtTypeId>();
        if (ftTypeId) {
            def.ftTypeId = ftTypeId;
        }
    }
}

void Loader::loadRubrics(PoiIdToDefMap& poiIdToDefMap)
{
    auto poiIdsWithoutAccurateRubricId =
        getPoiWithoutAccurateRubricId(poiIdToDefMap);
    if (poiIdsWithoutAccurateRubricId.empty()) {
        return;
    }
    std::stringstream query;
    query
        << "SELECT "
        << " poi." << col::FT_ID << " AS " << col::POI_FT_ID
        << ", poi." << col::FT_TYPE_ID << " AS " << col::POI_FT_TYPE_ID
        << ", parent." << col::FT_TYPE_ID  << " AS " << col::PARENT_FT_TYPE_ID
        << " FROM " << tbl::FT << " poi LEFT JOIN " << tbl::FT << " parent ON parent.ft_id=poi.p_ft_id"
        << " WHERE poi." << col::FT_ID << " IN ("
        << wiki::common::join(poiIdsWithoutAccurateRubricId, ",") << ")";
    auto rows = yMapsDf_.exec(query.str());

    for (const auto& row : rows) {
        auto poiId = row[col::POI_FT_ID].as<DBID>();
        auto& def = poiIdToDefMap[poiId];
        if (!row[col::PARENT_FT_TYPE_ID].is_null()) {
            auto parentFtTypeId = row[col::PARENT_FT_TYPE_ID].as<FtTypeId>();
            auto rng = parentMapping_.equal_range(parentFtTypeId);
            for (auto it = rng.first; it != rng.second; ++it) {
                def.rubricIds.insert(it->second);
            }
            if (rng.first != rng.second) {
                continue;
            }
        }
        auto ftTypeId = row[col::POI_FT_TYPE_ID].as<FtTypeId>();
        if (ftTypeId) {
            def.ftTypeId = ftTypeId;
        }
        auto rng = mapping_.equal_range(ftTypeId);
        for (auto it = rng.first; it != rng.second; ++it) {
            def.rubricIds.insert(it->second);
        }
    }
}

void Loader::loadNames(PoiIdToDefMap& poiIdToDefMap)
{
    std::stringstream query;
    query << "SELECT " << col::NM_ID << ", " << col::FT_ID << ", "
          << col::NAME << ", " << col::NAME_TYPE << ", " << col::LANG
          << " FROM " << tbl::FT_NM << " WHERE " << col::FT_ID << " IN ("
          << wiki::common::join(getPoiIds(poiIdToDefMap), ",") << ")";
    auto rows = yMapsDf_.exec(query.str());

    for (const auto& row : rows) {
        auto poiId = row[col::FT_ID].as<DBID>();
        auto& def = poiIdToDefMap[poiId];
        auto lang = row[col::LANG].as<std::string>();
        auto name = row[col::NAME].as<std::string>();
        switch ((NameType)row[col::NAME_TYPE].as<int>()) {
        case NameType::Official:
            def.names.insert({std::move(lang), std::move(name)});
            break;
        case NameType::RenderLabel:
            def.shortNames.insert({std::move(lang), std::move(name)});
            break;
        case NameType::Synonym:
            def.synonymNames.insert({std::move(lang), std::move(name)});
            break;
        }
        def.nameIds.insert(row[col::NM_ID].as<DBID>());
    }
}

void Loader::loadGeoms(PoiIdToDefMap& poiIdToDefMap)
{
    std::stringstream query;
    // clang-format off
    query << "SELECT "
              << col::FT_ID << ", "
              << "ST_X(ST_Centroid(" << col::SHAPE << ")) AS " << alias::X << ", "
              << "ST_Y(ST_Centroid(" << col::SHAPE << ")) AS " << alias::Y << " "
          << "FROM " << tbl::FT_CENTER << " "
          << "JOIN " << tbl::NODE << " USING (" << col::NODE_ID << ") "
          << "WHERE " << col::FT_ID << " IN ("
              << wiki::common::join(getPoiIds(poiIdToDefMap), ",") << ")";
    // clang-format on
    auto rows = yMapsDf_.exec(query.str());

    for (const auto& row : rows) {
        auto poiId = row[col::FT_ID].as<DBID>();
        auto& def = poiIdToDefMap[poiId];
        def.lon = row[alias::X].as<double>();
        def.lat = row[alias::Y].as<double>();
    }
}

void Loader::loadAttrs(PoiIdToDefMap& poiIdToDefMap)
{
    std::stringstream query;
    query << "SELECT " << col::FT_ID << ", " << col::KEY << ", " << col::VALUE
          << " FROM " << tbl::FT_ATTR << " WHERE " << col::FT_ID << " IN ("
          << common::join(getPoiIds(poiIdToDefMap), ",") << ")";
    auto rows = yMapsDf_.exec(query.str());

    for (const auto& row : rows) {
        const auto poiId = row[col::FT_ID].as<DBID>();
        auto& def = poiIdToDefMap[poiId];
        const auto key = row[col::KEY].as<std::string>();
        const auto value = row[col::VALUE].as<std::string>();
        if (key == "email") {
            def.email = value;
        } else if (key == "url") {
            def.url = value;
        } else if (key == "business_id") {
            if (!value.empty()) {
                if (isValidPermalink(value)) {
                    def.permalink = value;
                }
                else {
                    WARN() << "Found invalid permalink: [" << value
                           << "] for object " << poiId;
                }
            }
        } else if (key == "phone") {
            std::vector<std::string> phones;
            boost::split(phones, value, boost::is_any_of(","),
                            boost::token_compress_on);
            for (auto& phone : phones) {
                boost::trim(phone);
                def.phones.insert(phone);
            }
        } else if (key == "business_rubric_id") {
            def.userDefinedRubricId = boost::lexical_cast<RubricId>(value);
        } else if (key == "secondary_business_rubric_ids") {
            if (!value.empty()) {
                const auto secondaryRubricIds =
                    common::split(value, MULTIVALUE_DELIMITER);
                for (const auto& rubricId : secondaryRubricIds) {
                    def.userDefinedSecondaryRubricIds.insert(
                        boost::lexical_cast<RubricId>(rubricId));
                }
            }
        } else if (key == "open_hours") {
            def.workingTime = value;
        } else if (key == "open_type") {
            def.publishingStatus = publishingStatus(value, poiId);
        } else if (key == "mtr_id") {
            def.mtrId = value;
        } else if (key == "import_source") {
            def.importSource = value;
        } else if (key == "indoor_level_universal") {
            def.indoorLevelUniversal = value;
        } else if (key == "indoor_plan_id") {
            def.indoorPlanId = boost::lexical_cast<DBID>(value);
        } else if (key == "is_geoproduct") {
            def.isGeoproduct = poi_feed::FeedObjectData::IsGeoproduct::Yes;
        } else if (key == POSITION_QUALITY_FT_ATTR) {
            def.positionQuality = value;
            if (def.hasVerifiedCoords != HasVerifiedCoords::True &&
                poi_feed::isVerifiedPositionQuality(value))
            {
                def.hasVerifiedCoords = HasVerifiedCoords::True;
            }
        }
    }
    loadRubrics(poiIdToDefMap);
}

void Loader::loadEntrances(PoiIdToDefMap& poiIdToDefMap)
{
    std::stringstream query;
    // clang-format off
    query << "WITH " << tbl::ENTRANCES << " AS ( "
              << "SELECT "
                  << col::FT_ID << " AS " << col::POI_FT_ID << ", "
                  << "CAST(" << col::KEY << " AS bigint) AS " << col::ENTRANCE_ID << " "
              << "FROM " << tbl::FT_ATTR << " "
              << "WHERE " << col::VALUE << " = 'entrance_assigned' "
              << "AND " << col::FT_ID << " IN ("
              << common::join(getPoiIds(poiIdToDefMap), ",") << ")"
          << ") "
          << "SELECT "
              << tbl::ENTRANCES << "." << col::POI_FT_ID << " AS " << col::POI_FT_ID << ", "
              << col::FT_ID << ", "
              << "ST_X(ST_Centroid(" << col::SHAPE << ")) AS " << alias::X << ", "
              << "ST_Y(ST_Centroid(" << col::SHAPE << ")) AS " << alias::Y << " "
          << "FROM " << tbl::ENTRANCES << " "
          << "JOIN " << tbl::FT_CENTER << " ON ("
              << col::FT_ID << " = " << col::ENTRANCE_ID << ") "
          << "JOIN " << tbl::NODE << " USING (" << col::NODE_ID << ")";
    // clang-format on
    auto rows = yMapsDf_.exec(query.str());

    for (const auto& row : rows) {
        auto poiId = row[col::POI_FT_ID].as<DBID>();
        auto entranceId = row[col::FT_ID].as<DBID>();
        auto& poiDef = poiIdToDefMap[poiId];
        if (!poiDef.entrances) {
            poiDef.entrances = std::vector<EntranceDef>{};
        }
        poiDef.entrances->push_back({
            entranceId,
            row[alias::X].as<double>(),
            row[alias::Y].as<double>()});
    }

    auto deletedEntrancesFilter = revision::filters::ProxyFilterExpr(
        revision::filters::Attr(REL_ROLE_ATTR).equals(ROLE_ENTRANCE_ASSIGNED) &&
        revision::filters::ObjRevAttr::isDeleted());
    DBIDSet poiIds;
    for (const auto& poiIdToDef : poiIdToDefMap) {
        if (isFullMergeImplemented(poiIdToDef.second)) {
            poiIds.insert(poiIdToDef.first);
        }
    }
    auto relations = tds_.loadSlaveRelations(
        poiIds, deletedEntrancesFilter);
    for (const auto& relation : relations) {
        const auto poiId = relation.data().relationData->masterObjectId();
        auto& poiDef = poiIdToDefMap[poiId];
        if (!poiDef.entrances) {
            poiDef.entrances = std::vector<EntranceDef>{};
        }
    }
}

revision::RevisionIds
Loader::loadRevisionIds(const PoiIdToDefMap& poiIdToDefMap)
{
    DBIDSet dbIds;
    for (const auto& [id, poiDef] : poiIdToDefMap) {
        dbIds.insert(id);
        auto& nameIds = poiDef.nameIds;
        dbIds.insert(nameIds.begin(), nameIds.end());
    }
    return tds_.objectRevisionIds(dbIds);
}

void
Loader::updateWithRecentSlaveRevisions(
    const PoiIdToDefMap& poiIdToDefMap,
    ObjectIdToCommitIdMap& objectIdToMaxCommitId)
{
    const auto poiIds = getPoiIds(poiIdToDefMap);
    updateMapWithRecentSlaveRevisions(poiIds, tds_, objectIdToMaxCommitId);
}

ObjectIdCommitDataFunctor
Loader::loadObjectCommitDatas(const revision::RevisionIds& revIds, const PoiIdToDefMap& poiIdToDefMap)
{
    using CommitIdCommitDataMap = std::unordered_map<DBID, CommitData>;

    ObjectIdToCommitIdMap objectIdToCommitIdMap;
    DBIDSet commitIds;
    for (const auto& revId : revIds) {
        objectIdToCommitIdMap.emplace(revId.objectId(), revId.commitId());
    }
    updateWithRecentSlaveRevisions(poiIdToDefMap, objectIdToCommitIdMap);
    for (const auto& [objectId, commitId] : objectIdToCommitIdMap) {
        commitIds.insert(commitId);
    }

    CommitIdCommitDataMap commitIdCommitDataMap;
    auto commitFilter = revision::filters::CommitAttr::id().in(commitIds);
    auto commits = revision::Commit::load(tds_.reader().work(), commitFilter);
    for (const auto& commit : commits) {
        auto spravActualizationTimeIt =
            commit.attributes().find(poi_feed::COMMIT_ATTRIBUTE_SPRAV_ACTUALIZATION_DATE);
        chrono::TimePoint time = spravActualizationTimeIt != commit.attributes().end()
            ? chrono::parseSqlDateTime(spravActualizationTimeIt->second)
            : commit.createdAtTimePoint();
        commitIdCommitDataMap.emplace(
            commit.id(),
            CommitData {
                commit.id(),
                time,
                gdpr::User(commit.createdBy()).realUid()
            });
    }

    return [
        objectIdToCommitIdMap = std::move(objectIdToCommitIdMap),
        commitIdCommitDataMap = std::move(commitIdCommitDataMap)
    ](DBID objectId) mutable
    {
        return commitIdCommitDataMap[objectIdToCommitIdMap[objectId]];
    };
}

void Loader::loadActualizationDate(PoiIdToDefMap& poiIdToDefMap)
{
    auto revIds = loadRevisionIds(poiIdToDefMap);
    auto toCommitData = loadObjectCommitDatas(revIds, poiIdToDefMap);
    for (auto& poiIdToDef : poiIdToDefMap) {
        auto& def = poiIdToDef.second;
        const auto& commitData = toCommitData(def.id.get());
        def.recentCommitId = commitData.id;
        def.actualizationDate = commitData.time;
        def.modifiedBy = commitData.author;
        for (auto nameId : def.nameIds) {
            const auto& nameCommitData = toCommitData(nameId);
            if (def.actualizationDate.get() < nameCommitData.time) {
                def.recentCommitId = nameCommitData.id;
                def.actualizationDate = nameCommitData.time;
                def.modifiedBy = nameCommitData.author;
            }
        }
    }
}

void Loader::trySetPermalinkFromUnknown(PoiIdToDefMap& poiIdToDefMap)
{
    if (nyakMappingUnknownPermalinks_.empty()) {
        return;
    }
    for (auto& [id, def] : poiIdToDefMap) {
        if (!def.permalink.empty()) {
            continue;
        }
        auto it = nyakMappingUnknownPermalinks_.find(id);
        if (it != nyakMappingUnknownPermalinks_.end()) {
            def.permalink = std::to_string(it->second);
        }
    }
}

void Loader::resetReservedPermalinkForRealAssigned(const PoiIdToDefMap& poiIdToDefMap)
{
    std::unordered_set<poi_feed::ObjectId> hasRealPermalink;
    for (auto& [id, def] : poiIdToDefMap) {
        if (!def.permalink.empty()) {
            hasRealPermalink.insert(id);
        }
    }
    if (hasRealPermalink.empty()) {
        return;
    }
    common::retryDuration([&] {
        auto socialTxn = socialPool_.masterWriteableTransaction();
        poi_feed::clearAssignmentsOfReservedPermalinkId(*socialTxn, hasRealPermalink);
        socialTxn->commit();
    });
}

void Loader::setPermalinkFromReserve(PoiIdToDefMap& poiIdToDefMap)
{
    std::unordered_set<poi_feed::ObjectId> needPermalink;
    for (auto& [id, def] : poiIdToDefMap) {
        if (isFullMergeImplemented(def) && def.permalink.empty()) {
            needPermalink.insert(id);
        }
    }
    const auto objectIdToPermalinkId = common::retryDuration([&] {
        auto socialTxn = socialPool_.masterWriteableTransaction();
        auto objectIdToPermalinkId = poi_feed::assignReservedPermalinkId(*socialTxn, needPermalink);
        socialTxn->commit();
        return objectIdToPermalinkId;
    });
    for (const auto [id, permalinkId] : objectIdToPermalinkId) {
        poiIdToDefMap[id].permalink = std::to_string(permalinkId);
        poiIdToDefMap[id].permalinkFromReserves = PermalinkFromReserves::True;
    }
}

PoiDefs Loader::loadPoiDefs(const DBIDSet& poiIds)
{
    PoiIdToDefMap poiIdToDefMap;
    for (auto poiId : poiIds) {
        PoiDef def;
        def.id = poiId;
        poiIdToDefMap.emplace(poiId, def);
    }
    loadFtData(poiIdToDefMap);
    loadNames(poiIdToDefMap);
    loadGeoms(poiIdToDefMap);
    loadAttrs(poiIdToDefMap);
    loadEntrances(poiIdToDefMap);
    loadActualizationDate(poiIdToDefMap);
    resetReservedPermalinkForRealAssigned(poiIdToDefMap);
    setPermalinkFromReserve(poiIdToDefMap);

    PoiDefs result;
    result.reserve(poiIdToDefMap.size());
    for (auto& poiIdToDef : poiIdToDefMap) {
        result.push_back(std::move(poiIdToDef.second));
    }
    return result;
}

} // namespace maps::wiki::poi
