#include <util/generic/hash.h>
#include <util/generic/size_literals.h>

#include <mapreduce/yt/interface/protos/yamr.pb.h>

#include <robot/library/yt/static/command.h>
#include <robot/library/yt/static/tags.h>
#include <robot/jupiter/protos/acceptance.pb.h>
#include <robot/jupiter/protos/export.pb.h>

#include <yweb/antispam/common/owner/owner.h>
#include <yweb/robot/dbscheeme/urlflags.h>

#include <wmconsole/version3/library/jupiter/jupiter.h>
#include <wmconsole/version3/processors/tools/IKS/conf/config.h>
#include <wmconsole/version3/processors/tools/IKS/protos/iks.pb.h>
#include <wmconsole/version3/protos/queries2.pb.h>
#include <wmconsole/version3/wmcutil/log.h>

#include "utils.h"
#include "vitality.h"

using namespace NJupiter;

namespace NWebmaster {
namespace NIks {

TInputTag<NProto::TIKS> IKSInputTag                                     (0);
TInputTag<NProto::TIKSVitalityBandetector> VitalityBandetectorInputTag  (1);
TInputTag<NProto::TIKSVitalityHostdat> VitalityHostdatInputTag          (2);
TInputTag<NProto::TIKSVitalityRobot> VitalityRobotInputTag              (3);
TInputTag<NProto::TIKSVitalitySerp> VitalitySerpInputTag                (4);

TOutputTag<NProto::TIKSVitality> VitalityToBandetectorOutputTag         (0);
TOutputTag<NProto::TIKSVitality> VitalityOutputTag                      (1);

struct TVitalityMapper : public NYT::IMapper<NYT::TTableReader<NJupiter::TAcceptanceUrlForWebMasterRecord>, NYT::TTableWriter<NProto::TIKSVitalityRobot>> {
    Y_SAVELOAD_JOB(IKSOwners)

public:
    TVitalityMapper() = default;
    TVitalityMapper(const THashSet<TString> &iksOwners)
        : IKSOwners(iksOwners)
    {
    }

    void Start(TWriter *) override {
    }

    void Do(TReader *input, TWriter *output) override {
        TMascotOwnerCanonizer MascotOwnerCanonizer;
        MascotOwnerCanonizer.LoadTrueOwners();

        struct TRecord {
            size_t TotalUrls = 0;
            size_t GoodUrls = 0;
        };

        THashMap<TString, TRecord> records;
        for (; input->IsValid(); input->Next()) {
            const auto &row = input->GetRow();
            TString owner = MascotOwnerCanonizer.GetHostOwner(row.GetHost());
            if (owner.StartsWith("www.")) {
                owner = owner.substr(4);
            }
            if (!IKSOwners.contains(owner)) {
                continue;
            }
            records[owner].TotalUrls++;

            bool goodCode = false;
            bool redirectCode = false;
            if (row.GetHttpCode() >= 200 && row.GetHttpCode() < 300) {
                goodCode = true;
            } else if (row.GetHttpCode() >= 2000 && row.GetHttpCode() < 3000) {
                if (row.GetHttpCode() != 2014 /*EXT_HTTP_EMPTYDOC*/ && row.GetHttpCode() != 2024 /*EXT_HTTP_EMPTY_RESPONSE*/) {
                    goodCode = true;
                }
            } else if (row.GetHttpCode() >= 300 && row.GetHttpCode() < 400) {
                redirectCode = true;
            }

            if (goodCode || (!row.GetIsFake() && row.GetIsSearchable() && redirectCode)) {
                records[owner].GoodUrls++;
            }
        }

        for (const auto &obj : records) {
            NProto::TIKSVitalityRobot msg;
            msg.SetHost(obj.first);
            msg.SetUrlsTotal(obj.second.TotalUrls);
            msg.SetUrlsGood(obj.second.GoodUrls);
            output->AddRow(msg);
        }
    }

public:
    THashSet<TString> IKSOwners;
};

REGISTER_MAPPER(TVitalityMapper)

//ReduceBy Host
struct TVitalityReducer : public NYT::IReducer<NYT::TTableReader<NProto::TIKSVitalityRobot>, NYT::TTableWriter<NProto::TIKSVitalityRobot>> {
    void Do(TReader *input, TWriter *output) override {
        const TString host = input->GetRow().GetHost();
        size_t totalUrls = 0;
        size_t goodUrls = 0;
        for (; input->IsValid(); input->Next()) {
            const auto &row = input->GetRow();
            totalUrls += row.GetUrlsTotal();
            goodUrls += row.GetUrlsGood();
        }
        NProto::TIKSVitalityRobot msg;
        msg.SetHost(host);
        msg.SetUrlsTotal(totalUrls);
        msg.SetUrlsGood(goodUrls);
        output->AddRow(msg);
    }
};

REGISTER_REDUCER(TVitalityReducer)

struct TVitalityHostdatMapper : public NYT::IMapper<NYT::TTableReader<NJupiter::TAcceptanceHostRecord>, NYT::TTableWriter<NProto::TIKSVitalityHostdat>> {
    Y_SAVELOAD_JOB(IKSOwners)

public:
    TVitalityHostdatMapper() = default;
    TVitalityHostdatMapper(const THashSet<TString> &iksOwners)
        : IKSOwners(iksOwners)
    {
    }

    void Start(TWriter *) override {
    }

    void Do(TReader *input, TWriter *output) override {
        TMascotOwnerCanonizer MascotOwnerCanonizer;
        MascotOwnerCanonizer.LoadTrueOwners();

        struct TRecord {
            size_t OkStatusCount = 0;
            size_t BadStatusCount = 0;
        };

        THashMap<TString, TRecord> records;
        for (; input->IsValid(); input->Next()) {
            const auto &row = input->GetRow();
            TString owner = MascotOwnerCanonizer.GetHostOwner(row.GetHost());
            if (owner.StartsWith("www.")) {
                owner = owner.substr(4);
            }
            if (!IKSOwners.contains(owner)) {
                continue;
            }

            switch (row.GetHostStatus()) {
            case HostStatus::DNS_ERROR:
            case HostStatus::CONNECT_FAILED:
                records[owner].BadStatusCount++;
                break;
            default:
                records[owner].OkStatusCount++;
                break;
            }
        }

        for (const auto &obj : records) {
            NProto::TIKSVitalityHostdat msg;
            msg.SetHost(obj.first);
            msg.SetHostStatusCountOk(obj.second.OkStatusCount);
            msg.SetHostStatusCountBad(obj.second.BadStatusCount);
            output->AddRow(msg);
        }
    }

public:
    THashSet<TString> IKSOwners;
};

REGISTER_MAPPER(TVitalityHostdatMapper)

//ReduceBy Host
struct TVitalityHostdatReducer : public NYT::IReducer<NYT::TTableReader<NProto::TIKSVitalityHostdat>, NYT::TTableWriter<NProto::TIKSVitalityHostdat>> {
    void Do(TReader *input, TWriter *output) override {
        const TString host = input->GetRow().GetHost();
        size_t okStatusCount = 0;
        size_t badStatusCount = 0;
        for (; input->IsValid(); input->Next()) {
            const auto &row = input->GetRow();
            okStatusCount += row.GetHostStatusCountOk();
            badStatusCount += row.GetHostStatusCountBad();
        }
        NProto::TIKSVitalityHostdat msg;
        msg.SetHost(host);
        msg.SetHostStatusCountOk(okStatusCount);
        msg.SetHostStatusCountBad(badStatusCount);
        output->AddRow(msg);
    }
};

REGISTER_REDUCER(TVitalityHostdatReducer)

struct TQueriesMapper : public NYT::IMapper<NYT::TTableReader<NYT::TYamr>, NYT::TTableWriter<NProto::TIKSVitalitySerp>> {
    Y_SAVELOAD_JOB(IKSOwners)

public:
    TQueriesMapper() = default;
    TQueriesMapper(const THashSet<TString> &iksOwners)
        : IKSOwners(iksOwners)
    {
    }

    void Start(TWriter *) override {
    }

    void Do(TReader *input, TWriter *output) override {
        TMascotOwnerCanonizer MascotOwnerCanonizer;
        MascotOwnerCanonizer.LoadTrueOwners();

        struct TRecord {
            size_t Clicks = 0;
            size_t Shows = 0;
        };

        THashMap<TString, TRecord> records;
        for (; input->IsValid(); input->Next()) {
            const auto &row = input->GetRow();
            TString owner = MascotOwnerCanonizer.GetHostOwner(row.GetKey());
            if (owner.StartsWith("www.")) {
                owner = owner.substr(4);
            }
            if (!IKSOwners.contains(owner)) {
                continue;
            }

            TRecord &record = records[owner];
            proto::queries2::QueryMessage msg;
            Y_PROTOBUF_SUPPRESS_NODISCARD msg.ParseFromString(row.GetValue());

            for (int i = 0; i < msg.reports_by_region_size(); i++) {
                const auto &region = msg.reports_by_region(i);
                for (int p = 0; p < region.position_info_size(); p++) {
                     const auto &position = msg.reports_by_region(i).position_info(p);
                     record.Clicks += position.clicks_count();
                     record.Shows += position.shows_count();
                }
            }
        }

        for (const auto &obj : records) {
            NProto::TIKSVitalitySerp msg;
            msg.SetHost(obj.first);
            msg.SetSerpClicks(obj.second.Clicks);
            msg.SetSerpShows(obj.second.Shows);
            output->AddRow(msg);
        }
    }

public:
    THashSet<TString> IKSOwners;
};

REGISTER_MAPPER(TQueriesMapper)

//ReduceBy Host
struct TQueriesReducer : public NYT::IReducer<NYT::TTableReader<NProto::TIKSVitalitySerp>, NYT::TTableWriter<NProto::TIKSVitalitySerp>> {
    void Do(TReader *input, TWriter *output) override {
        const TString host = input->GetRow().GetHost();
        size_t clicks = 0;
        size_t shows = 0;
        for (; input->IsValid(); input->Next()) {
            const auto &row = input->GetRow();
            clicks += row.GetSerpClicks();
            shows += row.GetSerpShows();
        }
        NProto::TIKSVitalitySerp msg;
        msg.SetHost(host);
        msg.SetSerpClicks(clicks);
        msg.SetSerpShows(shows);
        output->AddRow(msg);
    }
};

REGISTER_REDUCER(TQueriesReducer)

//ReduceBy Host
struct TFindDeadOwnersReducer : public TTaggedReducer {
    void DoTagged(TTagedReader reader, TTagedWriter writer) override {
        TMaybe<NProto::TIKS> iksRow;
        TMaybe<NProto::TIKSVitalityBandetector> vitalityBandetectorRow;
        TMaybe<NProto::TIKSVitalityHostdat> vitalityHostdatRow;
        TMaybe<NProto::TIKSVitalityRobot> vitalityRobotRow;
        TMaybe<NProto::TIKSVitalitySerp> vitalitySerpRow;

        iksRow = reader.GetSingleRowMaybe(IKSInputTag);
        if (!iksRow.Defined()) {
            return;
        }

        size_t hostdatOkStatusCount = 0;
        size_t hostdatBadStatusCount = 0;
        size_t robotTotalUrls = 0;
        size_t robotGoodUrls = 0;
        size_t serpClicks = 0;
        size_t serpShows = 0;
        TString banType;

        if (reader.IsValid()) {
            vitalityBandetectorRow = reader.GetSingleRowMaybe(VitalityBandetectorInputTag);
        }

        if (reader.IsValid()) {
            vitalityHostdatRow = reader.GetSingleRowMaybe(VitalityHostdatInputTag);
        }

        if (reader.IsValid()) {
            vitalityRobotRow = reader.GetSingleRowMaybe(VitalityRobotInputTag);
        }

        if (reader.IsValid()) {
            vitalitySerpRow = reader.GetSingleRowMaybe(VitalitySerpInputTag);
        }

        if (vitalityBandetectorRow.Defined()) {
            banType = vitalityBandetectorRow.GetRef().GetBanType();
        }

        if (vitalityHostdatRow.Defined()) {
            hostdatOkStatusCount = vitalityHostdatRow.GetRef().GetHostStatusCountOk();
            hostdatBadStatusCount = vitalityHostdatRow.GetRef().GetHostStatusCountBad();
        }

        if (vitalityRobotRow.Defined()) {
            robotGoodUrls = vitalityRobotRow.GetRef().GetUrlsGood();
            robotTotalUrls = vitalityRobotRow.GetRef().GetUrlsTotal();
        }

        if (vitalitySerpRow.Defined()) {
            serpClicks = vitalitySerpRow.GetRef().GetSerpClicks();
            serpShows = vitalitySerpRow.GetRef().GetSerpShows();
        }

        const size_t iks = iksRow.GetRef().GetIKS();
        const bool isDead = (
            robotGoodUrls == 0
            && serpClicks == 0
            && serpShows == 0
        );

        const bool banned = !banType.empty();
        const bool isDeadWithoutBan = isDead && !banned;
        const bool needBanDetect = isDead
            && iks >= 20
            && hostdatOkStatusCount == 0
        ;

        NProto::TIKSVitality dstMsg;
        dstMsg.SetHost(iksRow.GetRef().GetHost());
        dstMsg.SetIKS(iksRow.GetRef().GetIKS());
        dstMsg.SetUrlsGood(robotGoodUrls);
        dstMsg.SetUrlsTotal(robotTotalUrls);
        dstMsg.SetSerpClicks(serpClicks);
        dstMsg.SetSerpShows(serpShows);
        dstMsg.SetHostStatusCountOk(hostdatOkStatusCount);
        dstMsg.SetHostStatusCountBad(hostdatBadStatusCount);

        if (needBanDetect) {
            writer.AddRow(dstMsg, VitalityToBandetectorOutputTag);
        }

        if (!banType.empty()) {
            dstMsg.SetBanType(banType);
        }

        dstMsg.SetIsDead(isDeadWithoutBan);
        writer.AddRow(dstMsg, VitalityOutputTag);
    }
};

REGISTER_REDUCER(TFindDeadOwnersReducer)

void UpdateVitalityBandetector(NYT::IClientBasePtr client) {
    TMap<TString, TString> results;
    auto reader = client->CreateTableReader<NYT::TNode>(TConfig::CInstance().TABLE_SOURCE_BANDETECTOR);
    auto writer = client->CreateTableWriter<NProto::TIKSVitalityBandetector>(
        NYT::TRichYPath(TConfig::CInstance().TABLE_IKS_VITALITY_FROM_BANDETECTOR).SortedBy("Host")
    );
    for (; reader->IsValid(); reader->Next()) {
        const auto &row = reader->GetRow();
        const auto &banTypeNode = row["ban_type"];
        if (!NYTUtils::IsNodeNull(banTypeNode)) {
            results[row["owner"].AsString()] = banTypeNode.AsString();
        }
    }
    NProto::TIKSVitalityBandetector msg;
    for (const auto &result : results) {
        msg.SetHost(result.first);
        msg.SetBanType(result.second);
        writer->AddRow(msg);
    }
    writer->Finish();
}

void UpdateVitality(NYT::IClientBasePtr client) {
    //Cout << GetJupiterAcceptanceInProdTable(client) << Endl;
    //THashMap<TString, size_t> ownerCY = {
    //    {"tehnosila.ru", 1100},
    //    {"bormental.ru", 750},
    //};

    const NYT::TSortColumns KEYS_IKS = {"Host"};

    THashSet<TString> ownerCY;
    LoadOwnerCY(client, TConfig::CInstance().TABLE_IKS_VITALITY_IKS, ownerCY);
    LoadOwnerCY(client, TConfig::CInstance().TABLE_IKS_DATA_ACCEPTED, ownerCY);

    TMapCombineReduceCmd<TVitalityMapper, TVitalityReducer, TVitalityReducer>(
        client,
        new TVitalityMapper(ownerCY),
        nullptr,
        new TVitalityReducer
    )
        .OperationWeight(TConfig::CInstance().OPERATION_WEIGHT)
        .MapperMemoryLimit(1_GBs)
        .Input(TTable<NJupiter::TAcceptanceUrlForWebMasterRecord>(client, DebugPath(GetJupiterAcceptanceInProdTable(client))))
        .Output(TTable<NProto::TIKSVitalityRobot>(client, TConfig::CInstance().TABLE_IKS_VITALITY_ROBOT))
        .ReduceBy(KEYS_IKS)
        .Do()
    ;

    TSortCmd<NProto::TIKSVitalityRobot>(client, TTable<NProto::TIKSVitalityRobot>(client, TConfig::CInstance().TABLE_IKS_VITALITY_ROBOT))
        .OperationWeight(TConfig::CInstance().OPERATION_WEIGHT)
        .By(KEYS_IKS)
        .Do()
    ;

    TDeque<NYTUtils::TTableInfo> sourceTables;
    if (!NYTUtils::GetTableList(client, TConfig::CInstance().TABLE_SOURCE_QUERIES_ROOT, sourceTables)) {
        LOG_ERROR("ERROR: No input tables found");
        return;
    }

    std::sort(sourceTables.rbegin(), sourceTables.rend(), NYTUtils::TTableInfo::TNameLess());
    if (sourceTables.size() > 30) {
        sourceTables.resize(30);
    }

    TDeque<TTable<NYT::TYamr>> inputTables;
    for (const NYTUtils::TTableInfo &table : sourceTables) {
        inputTables.push_back(TTable<NYT::TYamr>(client, table.Name));
    }

    TMapCombineReduceCmd<TQueriesMapper, TQueriesReducer, TQueriesReducer>(
        client, new TQueriesMapper(ownerCY), nullptr, new TQueriesReducer
    )
        .OperationWeight(TConfig::CInstance().OPERATION_WEIGHT)
        .Inputs(inputTables)
        .Output(TTable<NProto::TIKSVitalitySerp>(client, TConfig::CInstance().TABLE_IKS_VITALITY_SERP))
        .ReduceBy(KEYS_IKS)
        .MapperMemoryLimit(1_GBs)
        .Do()
    ;

    TSortCmd<NProto::TIKSVitalitySerp>(client, TTable<NProto::TIKSVitalitySerp>(client, TConfig::CInstance().TABLE_IKS_VITALITY_SERP))
        .OperationWeight(TConfig::CInstance().OPERATION_WEIGHT)
        .By(KEYS_IKS)
        .Do()
    ;

    TMapCombineReduceCmd<TVitalityHostdatMapper, TVitalityHostdatReducer, TVitalityHostdatReducer>(
        client,
        new TVitalityHostdatMapper(ownerCY),
        nullptr,
        new TVitalityHostdatReducer
    )
        .OperationWeight(TConfig::CInstance().OPERATION_WEIGHT)
        .MapperMemoryLimit(2_GBs)
        .Input(TTable<NJupiter::TAcceptanceHostRecord>(client, DebugPath(GetJupiterAcceptanceInProdHostTable(client))))
        .Output(TTable<NProto::TIKSVitalityHostdat>(client, TConfig::CInstance().TABLE_IKS_VITALITY_HOSTDAT))
        .ReduceBy(KEYS_IKS)
        .Do()
    ;

    TSortCmd<NProto::TIKSVitalityHostdat>(client, TTable<NProto::TIKSVitalityHostdat>(client, TConfig::CInstance().TABLE_IKS_VITALITY_HOSTDAT))
        .OperationWeight(TConfig::CInstance().OPERATION_WEIGHT)
        .By(KEYS_IKS)
        .Do()
    ;

    UpdateVitalityBandetector(client);

    TReduceCmd<TFindDeadOwnersReducer>(client)
        .OperationWeight(TConfig::CInstance().OPERATION_WEIGHT)
        .Input(TTable<NProto::TIKS>(client, TConfig::CInstance().TABLE_IKS_VITALITY_IKS), IKSInputTag)
        .Input(TTable<NProto::TIKSVitalityBandetector>(client, TConfig::CInstance().TABLE_IKS_VITALITY_FROM_BANDETECTOR), VitalityBandetectorInputTag)
        .Input(TTable<NProto::TIKSVitalityHostdat>(client, TConfig::CInstance().TABLE_IKS_VITALITY_HOSTDAT), VitalityHostdatInputTag)
        .Input(TTable<NProto::TIKSVitalityRobot>(client, TConfig::CInstance().TABLE_IKS_VITALITY_ROBOT), VitalityRobotInputTag)
        .Input(TTable<NProto::TIKSVitalitySerp>(client, TConfig::CInstance().TABLE_IKS_VITALITY_SERP), VitalitySerpInputTag)
        .Output(TTable<NProto::TIKSVitality>(client, TConfig::CInstance().TABLE_IKS_VITALITY_TO_BANDETECTOR).AsSortedOutput(KEYS_IKS), VitalityToBandetectorOutputTag)
        .Output(TTable<NProto::TIKSVitality>(client, TConfig::CInstance().TABLE_IKS_VITALITY_STATE).AsSortedOutput(KEYS_IKS), VitalityOutputTag)
        .ReduceBy(KEYS_IKS)
        .Do()
    ;
}

} //namespace NIks
} //namespace NWebmaster
