#include <util/draft/date.h>
#include <util/random/random.h>
#include <util/string/join.h>
#include <util/string/printf.h>

#include <mapreduce/yt/interface/protos/yamr.pb.h>

#include <robot/library/yt/static/command.h>
#include <robot/library/yt/static/table.h>
#include <robot/library/yt/static/tags.h>

#include <wmconsole/version3/processors/achievements/conf/config.h>
#include <wmconsole/version3/processors/achievements/protos/achievements.pb.h>
#include <wmconsole/version3/library/jupiter/jupiter.h>
#include <wmconsole/version3/processors/acceptance/protos/acceptance.pb.h>
#include <wmconsole/version3/searchqueries-mr/conf/yt.h>
#include <wmconsole/version3/processors/acceptance/conf/config.h>
#include <wmconsole/version3/processors/indexing/hostinfo/conf/config.h>
#include <wmconsole/version3/wmcutil/log.h>
#include <wmconsole/version3/wmcutil/yt/triggers.h>

#include "task_disallowed_urls.h"
#include "common.h"

namespace NWebmaster {
    namespace NAcceptance {

        NJupiter::TInputTag<NProto::TAchievementsDiffTmp> AchievementsDiffTmpInputTag               (0);

        NJupiter::TOutputTag<NProto::TAchievementsDiff> AchievementsDiffOutputTag                 (0);

        static NYT::TRichYPath DebugPath(const TString &table) {
            NYT::TRichYPath path(table);
//    path.AddRange(NYT::TReadRange().Exact(NYT::TReadLimit().Key(NYT::TKey("vega-ulitsa-gorbacheva.clients.site"))));
//    path.AddRange(NYT::TReadRange().Exact(NYT::TReadLimit().Key(NYT::TKey("http://khaliullin.info"))));
//    path.AddRange(NYT::TReadRange().Exact(NYT::TReadLimit().Key(NYT::TKey("https://petinaprokopova.rajce.idnes.cz"))));
//    path.AddRange(NYT::TReadRange().Exact(NYT::TReadLimit().Key(NYT::TKey({"https://cwetochki.ru"}))));
//    path.AddRange(NYT::TReadRange().Exact(NYT::TReadLimit().Key(NYT::TKey("https://www.drive2.ru"))));
//    path.AddRange(NYT::TReadRange().Exact(NYT::TReadLimit().Key(NYT::TKey("https://meshok.net"))));
//    path.AddRange(NYT::TReadRange().Exact(NYT::TReadLimit().Key(NYT::TKey("https://petskuafor.com"))));
            return NYT::TRichYPath(path);
        }

    struct TAchievementsDiffReducer : public NYT::IReducer<NYT::TTableReader<NYT::TNode>, NYT::TTableWriter<NYT::TNode>> {
        private:
            const ui32 TABLENO_CURRENT_TABLE = 0;
            const ui32 TABLENO_ACCEPTED_TABLE = 1;
            const ui32 TABLENO_DIFF_TABLE = 0;
            const ui32 TABLENO_NUMERIC_DIFF_TABLE = 1;
        public:
            TAchievementsDiffReducer() = default;

            void Do(TReader *input, TWriter *output) override {
                static NYT::TNode nullNode = NYT::TNode::CreateEntity();
                NYT::TNode prevState;
                NYT::TNode newState;

                for (const auto &cursor : *input) {
                    if (cursor.GetTableIndex() == TABLENO_CURRENT_TABLE) {
                        if (!cursor.GetRow().IsMap()){
                            continue;
                        }
                        prevState = cursor.GetRow();
                    } else if (cursor.GetTableIndex() == TABLENO_ACCEPTED_TABLE) {
                        if (!cursor.GetRow().IsMap()){
                            continue;
                        }
                        newState = cursor.GetRow();
                    } else {
                        ythrow yexception() << "expected reduce on 2 tables with index" << cursor.GetTableIndex();
                    }
                }

                TString tld;
                TString mascotOwner;
                THashMap<TString, int> statistics;
                TSet<TString> notNullColumns;
                THashMap<int, THashMap<TString, float>> numeric;


                if (newState.IsMap()) {
                    for (const auto &pair: newState.AsMap()) {
                        const auto &fieldName = pair.first;
                        if (fieldName == "MascotOwner") {
                            mascotOwner = pair.second.AsString();
                            continue;
                        }
                        if (fieldName == "Tld"){
                            tld = pair.second.AsString();
                            continue;
                        }
                        const NYT::TNode &prevValue =
                                prevState.IsMap() && prevState.HasKey(fieldName) ? prevState[fieldName] : nullNode;
                        const NYT::TNode &newValue =
                                newState.IsMap() && newState.HasKey(fieldName) ? newState[fieldName] : nullNode;
                        if (prevValue == nullNode && newValue != nullNode) {
                            statistics[fieldName] = NProto::EAchievementDiffType::APPEARED;
                        } else if (prevValue != nullNode && newValue == nullNode) {
                            statistics[fieldName] = NProto::EAchievementDiffType::DISAPPEARED;
                        } else if (prevValue != nullNode) {
                            if (prevValue.IsUint64() && prevValue != newValue) {
                                ui64 prev = prevValue.AsUint64();
                                ui64 curr = newValue.AsUint64();
                                if (prev > curr) {
                                    statistics[fieldName] = NProto::EAchievementDiffType::CHANGED_NUMERIC_DECREASE;
                                    numeric[NProto::EAchievementNumericChangeType::DECREASE][fieldName] = ((float)(prev - curr)) / prev;
                                } else {
                                    statistics[fieldName] = NProto::EAchievementDiffType::CHANGED_NUMERIC_INCREASE;
                                    numeric[NProto::EAchievementNumericChangeType::INCREASE][fieldName] = ((float)(curr - prev)) / prev;
                                }
                            } else if (prevValue != newValue) {
                                statistics[fieldName] = NProto::EAchievementDiffType::CHANGED;
                            }
                        }
                        if (prevValue != nullNode || newValue != nullNode) {
                            notNullColumns.insert(fieldName);
                        }
                    }
                } else if (prevState.IsMap()) {
                    for (const auto &pair: prevState.AsMap()) {
                        if (pair.first == "MascotOwner") {
                            mascotOwner = pair.second.AsString();
                            continue;
                        }
                        if (pair.first == "Tld"){
                            tld = pair.second.AsString();
                            continue;
                        }
                        const auto &fieldName = pair.first;
                        const NYT::TNode &prevValue =
                                prevState.IsMap() && prevState.HasKey(fieldName) ? prevState[fieldName] : nullNode;
                        if (prevValue != nullNode) {
                            statistics[fieldName] = NProto::EAchievementDiffType::DISAPPEARED;
                            notNullColumns.insert(fieldName);
                        }
                    }
                }

                if (!numeric.empty()) {
                    for (auto [type, diffs] : numeric) {
                        NYT::TNode dstMsg = NYT::TNode()
                                ("MascotOwner", mascotOwner)
                                ("Tld", tld)
                                ("Type", type);
                        for (auto &stats: diffs) {
                            dstMsg = dstMsg(stats.first, stats.second);
                        }
                        output->AddRow(dstMsg, TABLENO_NUMERIC_DIFF_TABLE);
                    }
                }

                for (auto &statsNotNull : notNullColumns){
                    NYT::TNode dstMsg = NYT::TNode();
                    output->AddRow(dstMsg
                                           ("Achievement", Sprintf("%s_%s", statsNotNull.c_str(), tld.c_str()))
                                           ("DiffCount", statistics.contains(statsNotNull) ? 1 : 0)
                                           ("NotNull", 1)
                                           ("DiffType", statistics.contains(statsNotNull) ? statistics[statsNotNull] : nullNode),
                                   TABLENO_DIFF_TABLE
                                   );
                }
            }
        };
        REGISTER_REDUCER(TAchievementsDiffReducer)

        struct TAchievementsDiffMergeReducer : public NJupiter::TTaggedReducer {
            TAchievementsDiffMergeReducer() = default;

            void DoTagged(NJupiter::TTagedReader reader, NJupiter::TTagedWriter writer) final {
                THashMap<::NWebmaster::NAcceptance::NProto::EAchievementDiffType, long> diffCount;
                long notNullCount = 0;
                TString name;

                for (auto &row : reader.GetRows(AchievementsDiffTmpInputTag)){
                    name = row.GetAchievement();
                    notNullCount += row.GetNotNull();
                    diffCount[row.GetDiffType()] += row.GetDiffCount();
                }

                if (notNullCount != 0) {
                    for (auto [type, diff] : diffCount) {
                        NProto::TAchievementsDiff dstMsg;
                        dstMsg.SetAchievement(name);
                        dstMsg.SetResultDiff(100.0 * diff / notNullCount);
                        dstMsg.SetDiffType(type);
                        writer.AddRow(dstMsg, AchievementsDiffOutputTag);
                    }
                }
            }
        };

        REGISTER_REDUCER(TAchievementsDiffMergeReducer)

        void ValidateAchievements(NYT::IClientBasePtr client, TString acceptanceTable, TString acceptanceNumericTable) {
            const THashMap<NProto::EAchievementDiffType, double> DIFF_THRESHOLD =
                    THashMap<NProto::EAchievementDiffType, double>({
                        std::pair(NProto::EAchievementDiffType::APPEARED, 30.0),
                        std::pair(NProto::EAchievementDiffType::DISAPPEARED, 30.0),
                        std::pair(NProto::EAchievementDiffType::CHANGED_NUMERIC_INCREASE, 45.0),
                        std::pair(NProto::EAchievementDiffType::CHANGED_NUMERIC_DECREASE, 45.0),
                        std::pair(NProto::EAchievementDiffType::CHANGED, 30.0)
                    });

            bool rejected = false;
            auto reader = NJupiter::TTable<NProto::TAchievementsDiff>(client, acceptanceTable).GetReader();
            for (; reader->IsValid(); reader->Next()) {
                const auto &row = reader->GetRow();
                if (row.GetResultDiff() > DIFF_THRESHOLD.at(row.GetDiffType())){
                    Cerr << row.GetAchievement() << Endl;
                    rejected = true;
                }
            }

            if (rejected) {
                ythrow yexception() << "ValidateAchievements are rejected";
            }

            auto numericReader = NJupiter::TTable<NYT::TNode>(client, acceptanceNumericTable).GetReader();
            THashMap<int, THashMap<TString, TDeque<double>>> diffs;
            THashMap<int, THashMap<TString, TMap<double, TString>>> samples;
            for (; numericReader->IsValid(); numericReader->Next()) {
                const auto &row = numericReader->GetRow();
                if (! row.IsMap()){
                    continue;
                }
                TString owner = row.AsMap().at("MascotOwner").AsString();
                int type = row.AsMap().at("Type").AsInt64();
                for (auto [columnName, columnValue]: row.AsMap()) {
                    if (!columnValue.IsDouble()) {
                        continue;
                    }
                    if (!diffs.contains(type)){
                        diffs[type] = THashMap<TString, TDeque<double>>();
                        samples[type] = THashMap<TString, TMap<double, TString>>();
                    }
                    if (!diffs.at(type).contains(columnName)){
                        diffs[type][columnName] = TDeque<double>();
                        samples[type][columnName] = TMap<double, TString>();
                    }
                    AddSample(diffs.at(type).at(columnName), samples.at(type).at(columnName), columnValue.AsDouble(), owner);
                }
            }

            for (auto [type, diffTyped]: diffs){
                for (auto [name, diffColumnNamed]: diffTyped){
                    rejected |= IsThresholdBroken(name, diffColumnNamed, samples.at(type).at(name), 0.1, 0.30);
                }
            }

            if (rejected) {
                ythrow yexception() << "ValidateAchievements are rejected";
            }
        }

        void ArchiveAchievementsAcceptance(NYT::IClientBasePtr client, TString acceptanceTable) {
            const auto &cfg = TConfig::CInstance();
            auto reader = client->CreateTableReader<NProto::TAchievementsDiff>(acceptanceTable);
            auto writer = client->CreateTableWriter<NProto::TAchievementsDiffArchive>(
                    NYT::TRichYPath(cfg.TABLE_ACCEPTANCE_ACHIEVEMENTS_ARCHIVE).Append(true)
            );
            int maxRowsCount = 10'000;
            int rowsCount = 0;

            NProto::TAchievementsDiffArchive dstMsg;

            for (; reader->IsValid() && rowsCount < maxRowsCount; reader->Next()) {
                const auto &row = reader->GetRow();
                if (row.GetResultDiff() != 0) {
                    dstMsg.SetAchievement(row.GetAchievement());
                    dstMsg.SetResultDiff(row.GetResultDiff());
                    dstMsg.SetDiffType(row.GetDiffType());
                    dstMsg.SetTimestamp(Now().TimeT());
                    writer->AddRow(dstMsg);
                    rowsCount++;
                }
            }
        }

        int AcceptAchievementsByTable(TString stageTable, TString acceptanceTable, TString acceptanceNumericTable, NYT::IClientBasePtr tx) {
            const auto &ccfg = TCommonYTConfig::CInstance();

            TString acceptedTable = ccfg.GetAcceptedPath(stageTable);

            if (!tx->Exists(acceptedTable)) {
                LOG_INFO("first launch, copying %s to %s", stageTable.c_str(), acceptedTable.c_str());
                tx->Copy(stageTable, acceptedTable, NYT::TCopyOptions().Recursive(true));
            }

            TYtSourceTrigger newStateTrigger(tx, stageTable);
            TYtSourceTrigger oldStateTrigger(tx, acceptanceTable);

            if (!oldStateTrigger.NeedUpdate(newStateTrigger.Source)) {
                LOG_INFO("source %s is already processed", stageTable.c_str());
                return 0;
            }

            if (tx->Exists(acceptanceTable)) {
                tx->Remove(acceptanceTable);
            }
            if (tx->Exists(acceptanceNumericTable)) {
                tx->Remove(acceptanceNumericTable);
            }

            NYT::TTableSchema acceptanceSchema;
            acceptanceSchema.Strict(true);
            acceptanceSchema.AddColumn(NYT::TColumnSchema().Name("Achievement").Type(NYT::VT_STRING));
            acceptanceSchema.AddColumn(NYT::TColumnSchema().Name("DiffCount").Type(NYT::VT_INT64));
            acceptanceSchema.AddColumn(NYT::TColumnSchema().Name("NotNull").Type(NYT::VT_INT64));
            acceptanceSchema.AddColumn(NYT::TColumnSchema().Name("DiffType").Type(NYT::VT_INT32));

            NYT::TTableSchema acceptanceNumericSchema;
            acceptanceNumericSchema.Strict(true);
            acceptanceNumericSchema.AddColumn(NYT::TColumnSchema().Name("MascotOwner").Type(NYT::VT_STRING));
            acceptanceNumericSchema.AddColumn(NYT::TColumnSchema().Name("Tld").Type(NYT::VT_STRING));
            acceptanceNumericSchema.AddColumn(NYT::TColumnSchema().Name("Type").Type(NYT::VT_INT32));

            NYT::TTableSchema achievementsSchema = NYT::CreateTableSchema(*::NWebmaster::NProto::TAchievements::GetDescriptor());
            NYT::TNode node = achievementsSchema.ToNode();

            for (auto column : node.AsList()) {
                TString name = column.AsMap()["name"].AsString();
                TString type = column.AsMap()["type"].AsString();
                if (type == "uint32" || type == "uint64" || type == "int32" || type == "int64"){
                    acceptanceNumericSchema.AddColumn(NYT::TColumnSchema().Name(name).Type(NYT::VT_FLOAT));
                }
            }

            NJupiter::TReduceCmd<TAchievementsDiffReducer>(tx)
                    .Input<NYT::TNode>(DebugPath(acceptedTable))
                    .Input<NYT::TNode>(DebugPath(stageTable))
                    .Output<NYT::TNode>(NYT::TRichYPath(acceptanceTable).Schema(acceptanceSchema))
                    .Output<NYT::TNode>(NYT::TRichYPath(acceptanceNumericTable).Schema(acceptanceNumericSchema))
                    .ReduceBy({"MascotOwner", "Tld"})
                    .Do();

            NJupiter::TSortCmd<NProto::TAchievementsDiffTmp>(tx)
                    .Input(NJupiter::TTable<NProto::TAchievementsDiffTmp>(tx, acceptanceTable))
                    .Output(NJupiter::TTable<NProto::TAchievementsDiffTmp>(tx, acceptanceTable))
                    .By({"Achievement"})
                    .Do();

            oldStateTrigger.Update(tx, newStateTrigger.Source);

            NJupiter::TReduceCmd<TAchievementsDiffMergeReducer>(tx)
                    .Input(NJupiter::TTable<NProto::TAchievementsDiffTmp>(tx, acceptanceTable), AchievementsDiffTmpInputTag)
                    .Output(NJupiter::TTable<NProto::TAchievementsDiff>(tx, acceptanceTable), AchievementsDiffOutputTag)
                    .ReduceBy({"Achievement"})
                    .Do();

            ValidateAchievements(tx, acceptanceTable, acceptanceNumericTable);
            ArchiveAchievementsAcceptance(tx, acceptanceTable);

            const auto opts = NYT::TCopyOptions().Force(true).Recursive(true);
            tx->Copy(stageTable, acceptedTable, opts);

            return 0;
        }

        int AcceptAchievementsRu(NYT::IClientBasePtr tx) {
            const auto &cfg = TConfig::CInstance();
            const auto &hicfg = NAchievements::TConfig::CInstance();

            TString stageTable = hicfg.TABLE_ACHIEVEMENTS_EXPORT_RU;
            TString acceptanceTable = cfg.TABLE_ACCEPTANCE_ACHIEVEMENTS_RU;
            TString acceptanceNumericTable = cfg.TABLE_ACCEPTANCE_ACHIEVEMENTS_RU_NUMERIC;

            return AcceptAchievementsByTable(stageTable, acceptanceTable, acceptanceNumericTable, tx);
        }

        int AcceptAchievementsKuub(NYT::IClientBasePtr tx) {
            const auto &cfg = TConfig::CInstance();
            const auto &hicfg = NAchievements::TConfig::CInstance();

            TString stageTable = hicfg.TABLE_ACHIEVEMENTS_EXPORT_KUUB;
            TString acceptanceTable = cfg.TABLE_ACCEPTANCE_ACHIEVEMENTS_KUUB;
            TString acceptanceNumericTable = cfg.TABLE_ACCEPTANCE_ACHIEVEMENTS_KUUB_NUMERIC;

            return AcceptAchievementsByTable(stageTable, acceptanceTable, acceptanceNumericTable, tx);
        }

        int AcceptAchievements(int, const char **) {
            const auto &cfg = TConfig::CInstance();
            NYT::IClientPtr client = NYT::CreateClient(cfg.MR_SERVER_HOST_ANTIALL);

            NYT::ITransactionPtr tx = client->StartTransaction();
//            auto tx = client;

            if (AcceptAchievementsRu(tx) == 0 && AcceptAchievementsKuub(tx) == 0) {
                tx->Commit();
                return 0;
            }
            tx->Abort();
            return 1;
        }
    } //namespace NAcceptance
} //namespace NWebmaster
