#include <util/digest/fnv.h>
#include <util/generic/hash_set.h>
#include <util/generic/size_literals.h>

#include <robot/library/yt/static/command.h>

#include <yweb/antispam/common/owner/owner.h>

#include <wmconsole/version3/library/jupiter/jupiter.h>
#include <wmconsole/version3/wmcutil/hostid.h>
#include <wmconsole/version3/wmcutil/thread.h>
#include <wmconsole/version3/wmcutil/yt/triggers.h>
#include <wmconsole/version3/processors/tools/host2vec/applier/protos/tables.pb.h>
#include <wmconsole/version3/processors/tools/host2vec/utils/utils.h>
#include <wmconsole/version3/processors/user_sessions/library/utils.h>
#include <wmconsole/version3/processors/user_sessions/niche/conf/config.h>
#include <wmconsole/version3/processors/user_sessions/niche/miner/tables.pb.h>
#include <wmconsole/version3/processors/user_sessions/protos/user_sessions.pb.h>

#include <wmconsole/version3/wmcutil/yt/misc.h>

#include <limits>

#include "task_mine_rival_queries.h"

namespace NWebmaster {
namespace NNiche {

using namespace NJupiter;

static const int H2V_LIMIT = 20;

static const TInputTag <NProto::TAddedRival> AddedRivalsInputTag(1);
static const TInputTag <NProto::THost2Vec> RivalH2VSpyLogInputTag(2);
static const TInputTag <NProto::THost2Vec> RivalH2VSimilarGroupInputTag(3);
static const TInputTag <NProto::TRivalAnalogy> RivalAnalogyInputTag(4);
static const TOutputTag <NProto::TRivalAnalogy> RivalAnalogyOutputTag(5);
static const TInputTag <NProto::THostSettings> HostSettingsInputTag(6);
static const TInputTag <NProto::TRivalHostQuery> RivalHostQueriesInputTag(7);
static const TOutputTag <NProto::TRivalHostQueryExtended> RivalHostQueriesExtendedOutputTag(8);
static const TInputTag <NProto::TRivalHostQueryExtended> RivalHostQueriesExtendedInputTag(9);
static const TInputTag <NProto::TMirror> MirrorInputTag(10);
static const TOutputTag <NProto::THostQuery> HostQueriesOutputTag(11);


struct TJoinHostsAndAddedRivalsReducer : public NYT::IReducer<NYT::TTableReader < NYT::TNode>,
                                         NYT::TTableWriter<NProto::TAddedRival>> {
public:
    void Do(TReader *input, TWriter *output) override {
        const TString mainHost = input->GetRow()["Host"].AsString();
        NProto::TAddedRival addedRival;
        addedRival.SetMainHost(mainHost);
        for (; input->IsValid(); input->Next()) {
            const TString rival(NUtils::GetHost2vecDomain(input->GetTableIndex() == 0 ? mainHost : input->GetRow()["Rival"].AsString()));
            addedRival.SetHost(rival);
            output->AddRow(addedRival);
        }
    }
};
REGISTER_REDUCER(TJoinHostsAndAddedRivalsReducer)

struct TRivalHost2VecReducer : public TTaggedReducer {
public:
    void DoTagged(TTagedReader reader, TTagedWriter writer) override {
        TDeque <TString> mainHosts;
        for (const NProto::TAddedRival &addedRival : reader.GetRows(AddedRivalsInputTag)) {
            mainHosts.push_back(addedRival.GetMainHost());
        }
        int count = 0;
        for (const NProto::THost2Vec &h2v : reader.GetRows(RivalH2VSpyLogInputTag)) {
            if (count++ > H2V_LIMIT) {
                break;
            }
            for (const TString &host : mainHosts) {
                NProto::TRivalAnalogy analogy;
                analogy.SetMainHost(host);
                analogy.SetHost(h2v.GetAnalogy());
                analogy.SetSource(NProto::HOST_2_VEC_SPY_LOG);
                writer.AddRow(analogy, RivalAnalogyOutputTag);
            }
        }
        reader.SkipRows(RivalH2VSpyLogInputTag);
        count = 0;
        for (const NProto::THost2Vec &h2v : reader.GetRows(RivalH2VSimilarGroupInputTag)) {
            if (count++ > H2V_LIMIT) {
                break;
            }
            for (const TString &host : mainHosts) {
                NProto::TRivalAnalogy analogy;
                analogy.SetMainHost(host);
                analogy.SetHost(h2v.GetAnalogy());
                analogy.SetSource(NProto::HOST_2_VEC_SIMILAR_GROUP);
                writer.AddRow(analogy, RivalAnalogyOutputTag);
            }
        }
    }
};

REGISTER_REDUCER(TRivalHost2VecReducer)

struct TRivalAnalogyFilterReducer : public TTaggedReducer {
public:
    void DoTagged(TTagedReader reader, TTagedWriter writer) override {
        TString mainHost;
        THashMap <TString, THashSet<NProto::EHost2VecSource>> rivalsSources;
        for (const NProto::TRivalAnalogy &rivalAnalogy : reader.GetRows(RivalAnalogyInputTag)) {
            if (mainHost.empty()) {
                mainHost = rivalAnalogy.GetMainHost();
            }
            rivalsSources[rivalAnalogy.GetHost()].insert(rivalAnalogy.GetSource());
        }
        TDeque <TString> filteredRivals;
        THashSet <TString> filterdRivalOwners;
        for (const auto &pair : rivalsSources) {
            if (pair.second.size() >= 2) {
                filteredRivals.push_back(pair.first);
                filterdRivalOwners.insert(MascotOwnerCanonizer.GetHostOwner(pair.first));
            }
        }
        for (const auto &rival : filteredRivals) {
            NProto::TRivalAnalogy row;
            row.SetMainHost(mainHost);
            row.SetHost(rival);
            row.SetRivalCount(filteredRivals.size());
            row.SetRivalOwnerCount(filterdRivalOwners.size());
            writer.AddRow(row, RivalAnalogyOutputTag);
        }
    }

private:
    TMascotOwnerCanonizer MascotOwnerCanonizer;
};

REGISTER_REDUCER(TRivalAnalogyFilterReducer)

struct TRivalMainMirrorsExtractMapper : public NYT::IMapper<NYT::TTableReader < NProto::TMirror>,
                                        NYT::TTableWriter<NProto::TMirror>> {
    Y_SAVELOAD_JOB(Domains)
public:
    TRivalMainMirrorsExtractMapper() = default;
    TRivalMainMirrorsExtractMapper(const THashSet <TString> &domains)
        : Domains(domains) {
    }

    void Do(TReader *input, TWriter *output) override {
        for (; input->IsValid(); input->Next()) {
            const auto &row = input->GetRow();
            if (row.GetMainHost() != row.GetHost()) {
                continue;
            }
            TString domain(NUtils::GetHost2vecDomain(row.GetMainHost()));
            if (Domains.contains(domain)) {
                NProto::TMirror dstMsg;
                dstMsg.SetHost(domain);
                dstMsg.SetMainHost(row.GetMainHost());
                dstMsg.SetRank(row.GetRank());
                output->AddRow(dstMsg);
            }
        }
    }
private:
    THashSet <TString> Domains;
};
REGISTER_MAPPER(TRivalMainMirrorsExtractMapper)

struct TPropagateRivalToMainMirrorReducer : public TTaggedReducer {
    void DoTagged(TTagedReader reader, TTagedWriter writer) override {
        TDeque <TString> mainMirrors;
        for (const auto &row : reader.GetRows(MirrorInputTag)) {
            mainMirrors.emplace_back(row.GetMainHost());
        }
        for (auto row : reader.GetRows(RivalAnalogyInputTag)) {
            for (const TString &mainMirror : mainMirrors) {
                row.SetHost(mainMirror);
                writer.AddRow(row, RivalAnalogyOutputTag);
            }
        }
    }
};

REGISTER_REDUCER(TPropagateRivalToMainMirrorReducer)

struct TUserSessionFilterAndReducer : public NYT::IReducer<NYT::TTableReader < NUserSessions::NProto::TQuery>,
                                      NYT::TTableWriter<NProto::TRivalHostQuery>> {
public:
    void Do(TReader *input, TWriter *output) override {
        NProto::TRivalHostQuery result;
        int count = 0;
        int position = 0;
        for (; input->IsValid(); input->Next()) {
            auto row = input->GetRow();
            if (row.GetCorrectedQuery().length()< 6 || !NUserSessions::IsVisibleQueryInWebmaster(row) ||
                row.GetUpperQueryNavPred() >= 0.5 || row.GetIsNav()== true || row.GetPosition()>= 50) {
                continue;
            }
            if (count == 0) {
                result.SetHost(row.GetHost());
                result.SetCm2(row.GetCm2());
                result.SetTableSource(row.GetTableSource());
                result.SetRegionId(row.GetRegionId());
                result.SetQuery(row.GetCorrectedQuery());
                result.SetPath(row.GetPath());
                result.SetPosition(row.GetPosition());
            }
            count++;
            position += row.GetPosition();
        }
        if (count > 0) {
            result.SetCount(count);
            result.SetInvCount(FnvHash<i64>(result.GetQuery()));
            result.SetPosition((double) position / count);
            output->AddRow(result);
        }
    }
};
REGISTER_REDUCER(TUserSessionFilterAndReducer)

struct TUserSessionLimitFilter : public NYT::IReducer<NYT::TTableReader < NProto::TRivalHostQuery>,
                                 NYT::TTableWriter<NProto::TRivalHostQuery>> {
    Y_SAVELOAD_JOB(Limit)
public:
    TUserSessionLimitFilter() = default;
    TUserSessionLimitFilter(int limit) : Limit(limit) {
    }

    void Do(TReader *input, TWriter *output) override {
        for (int i = 0; input->IsValid() && i<Limit; input->Next(), i++) {
            output->AddRow(input->GetRow());
        }
    }
private:
    int Limit = 0;
};
REGISTER_REDUCER(TUserSessionLimitFilter)

struct TRivalCrossJoinQueriesReducer : public TTaggedReducer {
    void DoTagged(TTagedReader reader, TTagedWriter writer) override {
        TDeque <NProto::TRivalAnalogy> rivals;
        for (auto row : reader.GetRows(RivalAnalogyInputTag)) {
            rivals.push_back(row);
        }
        NProto::TRivalHostQueryExtended result;
        for (auto row : reader.GetRows(RivalHostQueriesInputTag)) {
            result.SetHost(row.GetHost());
            result.SetCm2(row.GetCm2());
            result.SetTableSource(row.GetTableSource());
            result.SetRegionId(row.GetRegionId());
            result.SetQuery(row.GetQuery());
            result.SetPath(row.GetPath());
            result.SetPosition(row.GetPosition());
            result.SetCount(row.GetCount());
            for (const auto &rival : rivals) {
                result.SetMainHost(rival.GetMainHost());
                result.SetRivalCount(rival.GetRivalCount());
                result.SetRivalOwnerCount(rival.GetRivalOwnerCount());
                writer.AddRow(result, RivalHostQueriesExtendedOutputTag);
            }
        }
    }
};

REGISTER_REDUCER(TRivalCrossJoinQueriesReducer)

struct TMinedQueriesSettingsFilterReducer : public TTaggedReducer {
    void DoTagged(TTagedReader reader, TTagedWriter writer) override {
        NProto::THostQuery msg;
        NProto::TRivalHostQueryExtended row;
        THashSet <TString> rivals;
        THashSet <TString> rivalOwners;
        int count = 0;
        for (auto iterRow : reader.GetRows(RivalHostQueriesExtendedInputTag)) {
            row = iterRow;
            rivals.insert(row.GetHost());
            rivalOwners.insert(MascotOwnerCanonizer.GetHostOwner(row.GetHost()));
            count += row.GetCount();
        }

        if (rivalOwners.size() >= 3) {
            msg.SetHost(row.GetMainHost());
            msg.SetQuery(row.GetQuery());
            msg.SetCount(count);
            msg.SetInvCount(FnvHash<i64>(msg.GetQuery()));
            msg.SetRivalCount(-rivals.size());
            msg.SetOwnerCount(-rivalOwners.size());
            writer.AddRow(msg, HostQueriesOutputTag);
        }
    }

private:
    TMascotOwnerCanonizer MascotOwnerCanonizer;
};

REGISTER_REDUCER(TMinedQueriesSettingsFilterReducer)

struct THostQueryFilter : public NYT::IReducer<NYT::TTableReader <NProto::THostQuery>,
                          NYT::TTableWriter<NProto::THostQuery>> {
    Y_SAVELOAD_JOB(Limit)
public:
    THostQueryFilter() = default;
    THostQueryFilter(int limit) : Limit(limit) {
    }

    void Do(TReader *input, TWriter *output) override {
        const size_t TABLENO_QUERIES = 0;
        const size_t TABLENO_TOTAL_QUERIES = 1;
        ui64 totalCount = 0;
        TString host;
        for (int i = 0; input->IsValid() && i<Limit; input->Next(), i++) {
            const NProto::THostQuery &row = input->GetRow();
            if (host.empty()) {
                host = row.GetHost();
            }
            totalCount += row.GetCount();
            output->AddRow(row, TABLENO_QUERIES);
        }
        NProto::THostQuery result;
        result.SetHost(host);
        result.SetCount(totalCount);
        output->AddRow(result, TABLENO_TOTAL_QUERIES);
    }
private:
    int Limit = 0;
};
REGISTER_REDUCER(THostQueryFilter)

struct TJoinTotalQueriesCountReducer : public NYT::IReducer<NYT::TTableReader <NProto::THostQuery>,
                          NYT::TTableWriter<NProto::THostQuery>> {
public:
    void Do(TReader *input, TWriter *output) override {
        const size_t TABLENO_TOTAL_QUERIES = 0;
        if (input->GetTableIndex() != TABLENO_TOTAL_QUERIES) {
            return;
        }
        ui64 totalQueries = input->GetRow().GetCount();
        input->Next();
        for (int i = 0; input->IsValid(); input->Next(), i++) {
            NProto::THostQuery row = input->GetRow();
            row.SetShare(static_cast<double>(row.GetCount()) / totalQueries);
            output->AddRow(row);
        }
    }
};
REGISTER_REDUCER(TJoinTotalQueriesCountReducer)

void MineRivalQueries(int weekNumber) {
    const auto &config = TConfig::CInstance();
    NYT::IClientBasePtr client = NYT::CreateClient(config.MR_SERVER_HOST);
    auto tx = client->StartTransaction();
    LOG_INFO("Mining rival queries for week %i", weekNumber);
    TDeque <NYTUtils::TTableInfo> tables;
    NYTUtils::GetTableList(client, config.TABLE_USER_SESSION_ROOT, tables);
    std::sort(tables.begin(), tables.end(), NYTUtils::TTableInfo::TNameLess());
    TDeque <TTable<NUserSessions::NProto::TQuery>> userSessionsForProcess;
    for (auto table : tables) {
        time_t utcTime = 0;
        if (!ParseISO8601DateTime(NYTUtils::GetTableName(table.Name).c_str(), utcTime)) {
            continue;
        }
        int newWeekNumber = (utcTime / 86400 + 3) / 7;
        if (newWeekNumber == weekNumber) {
            LOG_INFO("Mining rival queries from user sessions %s", table.Name.c_str());
            userSessionsForProcess.push_back(TTable<NUserSessions::NProto::TQuery>(tx, table.Name));
        } else if (newWeekNumber > weekNumber) {
            break;
        }
    }
    LOG_INFO("Prepare Rival and UserSession.");

    auto generateRivalHosts = [&] {
        LOG_INFO("Prepare rivals table");
        const TString TABLE_TMP_ADDED_RIVALS = NYTUtils::JoinPath(config.TABLE_NICHE_ROOT, "tmp", "added_rivals");
        const TString TABLE_TMP_HOSTS_AND_ADDED_RIVALS = NYTUtils::JoinPath(config.TABLE_NICHE_ROOT, "tmp",
                                                                            "hosts_and_added_rivals");
        const TString TABLE_TMP_RIVAL_MIRRORS = NYTUtils::JoinPath(config.TABLE_NICHE_ROOT, "tmp", "rival_mirrors");
        auto writer = tx->CreateTableWriter<NYT::TNode>(TABLE_TMP_ADDED_RIVALS);
        for (auto reader = tx->CreateTableReader<NYT::TNode>(
            config.TABLE_SOURCE_ADDED_RIVALS); reader->IsValid(); reader->Next()) {
            const auto &row = reader->GetRow();
            const auto &rival = row["rival"].AsString();
            const auto host = TWebmasterHostId::FromHostId(row["host_id"].AsString()).ToHostName();
            writer->AddRow(NYT::TNode()("Host", host)("Rival", rival));
        }
        writer->Finish();
        TSortCmd<NYT::TNode>(tx)
            .Input<NYT::TNode>(TABLE_TMP_ADDED_RIVALS)
            .Output<NYT::TNode>(TABLE_TMP_ADDED_RIVALS)
            .By("Host")
            .Do();

        TReduceCmd<TJoinHostsAndAddedRivalsReducer>(tx)
            .Input<NYT::TNode>(config.TABLE_SOURCE_WEBMASTER_HOSTS)
            .Input<NYT::TNode>(TABLE_TMP_ADDED_RIVALS)
            .Output<NProto::TAddedRival>(TABLE_TMP_HOSTS_AND_ADDED_RIVALS)
            .ReduceBy("Host")
            .Do();

        TSortCmd<NProto::TAddedRival>(tx, TTable<NProto::TAddedRival>(tx, TABLE_TMP_HOSTS_AND_ADDED_RIVALS))
            .By("Host")
            .Do();

        LOG_INFO("Multiplying rivals by host2vec models");
        TReduceCmd<TRivalHost2VecReducer>(tx)
            .Input(TTable<NProto::TAddedRival>(tx, TABLE_TMP_HOSTS_AND_ADDED_RIVALS), AddedRivalsInputTag)
            .Input(TTable<NProto::THost2Vec>(tx, config.TABLE_SOURCE_H2V_SPYLOG), RivalH2VSpyLogInputTag)
            .Input(TTable<NProto::THost2Vec>(tx, config.TABLE_SOURCE_H2V_SIMILARGROUP), RivalH2VSimilarGroupInputTag)
            .Output(TTable<NProto::TRivalAnalogy>(tx, config.TABLE_TMP_RIVAL + "1"), RivalAnalogyOutputTag)
            .ReduceBy("Host")
            .Do();
        LOG_INFO("Sorting preprocessed rivals");
        TSortCmd<NProto::TRivalAnalogy>(tx, TTable<NProto::TRivalAnalogy>(tx, config.TABLE_TMP_RIVAL + "1"))
            .By("MainHost")
            .Do();
        LOG_INFO("Filtering rivals (save only hosts from both h2v)");
        TReduceCmd<TRivalAnalogyFilterReducer>(tx)
            .Input(TTable<NProto::TRivalAnalogy>(tx, config.TABLE_TMP_RIVAL + "1"), RivalAnalogyInputTag)
            .Output(TTable<NProto::TRivalAnalogy>(tx, config.TABLE_TMP_RIVAL + "2"), RivalAnalogyOutputTag)
            .ReduceBy("MainHost")
            .Do();

        LOG_INFO("Sorting filtered rivals");
        TSortCmd<NProto::TRivalAnalogy>(tx, TTable<NProto::TRivalAnalogy>(tx, config.TABLE_TMP_RIVAL + "2"))
            .By("Host")
            .Do();

        THashSet <TString> rivals;
        for (auto reader = tx->CreateTableReader<NProto::TRivalAnalogy>(
            config.TABLE_TMP_RIVAL + "2"); reader->IsValid(); reader->Next()) {
            const NProto::TRivalAnalogy &row = reader->GetRow();
            rivals.insert(row.GetHost());
        }
        LOG_INFO("Collecting main mirrors for rivals");
        TMapCmd<TRivalMainMirrorsExtractMapper>(tx, new TRivalMainMirrorsExtractMapper(rivals))
            .Input(TTable<NProto::TMirror>(tx, GetJupiterMirrorsInProdTable(tx)))
            .Output(TTable<NProto::TMirror>(tx, TABLE_TMP_RIVAL_MIRRORS))
            .MemoryLimit(2_GBs)
            .Do();

        TSortCmd<NProto::TMirror>(tx, TTable<NProto::TMirror>(tx, TABLE_TMP_RIVAL_MIRRORS))
            .By("Host")
            .Do();

        LOG_INFO("Propagating rivals to its main mirrors");
        TReduceCmd<TPropagateRivalToMainMirrorReducer>(tx)
            .Input(TTable<NProto::TMirror>(tx, TABLE_TMP_RIVAL_MIRRORS), MirrorInputTag)
            .Input(TTable<NProto::TRivalAnalogy>(tx, config.TABLE_TMP_RIVAL + "2"), RivalAnalogyInputTag)
            .Output(TTable<NProto::TRivalAnalogy>(tx, config.TABLE_TMP_RIVAL + "3"), RivalAnalogyOutputTag)
            .ReduceBy("Host")
            .Do();

        LOG_INFO("Sorting propagated rivals");
        TSortCmd<NProto::TRivalAnalogy>(tx, TTable<NProto::TRivalAnalogy>(tx, config.TABLE_TMP_RIVAL + "3"))
            .By("Host")
            .Do();
    };

    auto prepareQueries = [&] {
        LOG_INFO("Filtering user sessions");
        TReduceCmd<TUserSessionFilterAndReducer>(tx, new TUserSessionFilterAndReducer)
            .Inputs(userSessionsForProcess)
            .Output(TTable<NProto::TRivalHostQuery>(tx, config.TABLE_TMP_MINED_RIVAL_QUERIES + "1").AsSortedOutput(
                {"Host", "Query"}))
            .ReduceBy({"Host", "CorrectedQuery", "Path", "RegionId"})
            .Do();

        TSortCmd<NProto::TRivalHostQuery>(tx, TTable<NProto::TRivalHostQuery>(tx, config.TABLE_TMP_MINED_RIVAL_QUERIES +
                                                                                  "1"))
            .By({"Host", "InvCount"})
            .Do();

        LOG_INFO("Filter mined queries by limits");
        TCombineReduceCmd<TUserSessionLimitFilter, TUserSessionLimitFilter>(tx, new TUserSessionLimitFilter(1000000),
                                                                            new TUserSessionLimitFilter(1000000))
            .Input(TTable<NProto::TRivalHostQuery>(tx, config.TABLE_TMP_MINED_RIVAL_QUERIES + "1"))
            .Output(TTable<NProto::TRivalHostQuery>(tx, config.TABLE_TMP_MINED_RIVAL_QUERIES + "2"))
            .ReduceBy({"Host"})
            .SortBy({"Host", "InvCount"})
            .Do();

        LOG_INFO("Sort Queries by Host,Query,RegionId,TableSource");
        TSortCmd<NProto::TRivalHostQuery>(tx, TTable<NProto::TRivalHostQuery>(tx, config.TABLE_TMP_MINED_RIVAL_QUERIES +
                                                                                  "2"))
            .By({"Host", "Query", "RegionId", "TableSource"})
            .Do();
    };

    NUtils::RunAsync(generateRivalHosts, prepareQueries);

    LOG_INFO("Join rivals and queries");
    TReduceCmd<TRivalCrossJoinQueriesReducer>(tx, new TRivalCrossJoinQueriesReducer)
        .Input(TTable<NProto::TRivalAnalogy>(tx, config.TABLE_TMP_RIVAL + "3"), RivalAnalogyInputTag)
        .Input(TTable<NProto::TRivalHostQuery>(tx, config.TABLE_TMP_MINED_RIVAL_QUERIES + "2"),
               RivalHostQueriesInputTag)
        .Output(TTable<NProto::TRivalHostQueryExtended>(tx, config.TABLE_TMP_MINED_RIVAL_QUERIES + "3"),
                RivalHostQueriesExtendedOutputTag)
        .ReduceBy({"Host"})
        .JobCount(100000)
        .Do();

    LOG_INFO("Sort Queries by MainHost, Query,RegionId,TableSource");
    TSortCmd<NProto::TRival>(tx, TTable<NProto::TRival>(tx, config.TABLE_TMP_MINED_RIVAL_QUERIES + "3"))
        .By({"MainHost", "Query"})
        .Do();

    TReduceCmd<TMinedQueriesSettingsFilterReducer>(tx)
        .Input(TTable<NProto::TRivalHostQueryExtended>(tx, config.TABLE_TMP_MINED_RIVAL_QUERIES + "3"),
               RivalHostQueriesExtendedInputTag)
        .Output(TTable<NProto::THostQuery>(tx, config.TABLE_TMP_HOST_QUERY_DINAMIC_NICHE + "0"), HostQueriesOutputTag)
        .ReduceBy({"MainHost", "Query"})
        .Do();
    TSortCmd<NProto::THostQuery>(tx, TTable<NProto::THostQuery>(tx, config.TABLE_TMP_HOST_QUERY_DINAMIC_NICHE + "0"))
        .By({"Host", "InvCount"})
        .Do();

    const TString TABLE_TMP_TOTAL_QUERIES = NYTUtils::JoinPath(config.TABLE_NICHE_ROOT, "tmp", "total_queries_in_niche");

    TCombineReduceCmd<THostQueryFilter, THostQueryFilter>(tx, new THostQueryFilter(20000), new THostQueryFilter(20000))
        .Input(TTable<NProto::THostQuery>(tx, config.TABLE_TMP_HOST_QUERY_DINAMIC_NICHE + "0"))
        .Output(TTable<NProto::THostQuery>(tx, config.TABLE_TMP_HOST_QUERY_DINAMIC_NICHE + "1"))
        .Output(TTable<NProto::THostQuery>(tx, TABLE_TMP_TOTAL_QUERIES))
        .ReduceBy({"Host"})
        .SortBy({"Host", "InvCount"})
        .Do();

    DoParallel(
        TSortCmd<NProto::THostQuery>(tx, TTable<NProto::THostQuery>(tx, config.TABLE_TMP_HOST_QUERY_DINAMIC_NICHE + "1")).By({"Host"}),
        TSortCmd<NProto::THostQuery>(tx, TTable<NProto::THostQuery>(tx, TABLE_TMP_TOTAL_QUERIES)).By({"Host"})
    );

    TReduceCmd<TJoinTotalQueriesCountReducer>(tx)
        .Input(TTable<NProto::THostQuery>(tx, TABLE_TMP_TOTAL_QUERIES))
        .Input(TTable<NProto::THostQuery>(tx, config.TABLE_TMP_HOST_QUERY_DINAMIC_NICHE + "1"))
        .Output(TTable<NProto::THostQuery>(tx, config.TABLE_TMP_HOST_QUERY_DINAMIC_NICHE + "1"))
        .ReduceBy({"Host"})
        .Do();

    TSortCmd<NProto::THostQuery>(tx, TTable<NProto::THostQuery>(tx, config.TABLE_TMP_HOST_QUERY_DINAMIC_NICHE + "1"))
        .By({"Host", "Query"})
        .Do();

    NYTUtils::SetAttr(tx, config.TABLE_TMP_HOST_QUERY_DINAMIC_NICHE + "1", TAttrName::SourceWeekNumber, weekNumber);
    tx->Commit();
}

} // namespace NNiche
} // namespace NWebmaster
