#include <util/digest/fnv.h>
#include <util/draft/date.h>
#include <util/generic/deque.h>
#include <util/generic/hash_set.h>
#include <util/generic/set.h>
#include <util/generic/singleton.h>
#include <util/thread/pool.h>

#include <kernel/hosts/owner/owner.h>
#include <mapreduce/yt/interface/client.h>
#include <quality/logs/parse_lib/parse_lib.h>
#include <quality/logs/parse_lib/parsing_rules.h>
#include <quality/traffic/iterator/iterator.h>
//#include <quality/user_search/common/clicks_shows_stats.pb.h>
#include <yweb/antispam/mascot/proto/import.pb.h>

#include <wmconsole/version3/wmcutil/hostid.h>
#include <wmconsole/version3/wmcutil/log.h>
#include <wmconsole/version3/wmcutil/regex.h>
#include <wmconsole/version3/wmcutil/thread.h>
#include <wmconsole/version3/wmcutil/url.h>
#include <wmconsole/version3/wmcutil/yt/yt_runner.h>
#include <wmconsole/version3/wmcutil/yt/yt_utils.h>

#include <wmconsole/version3/processors/user_sessions/conf/yt.h>

namespace NWebmaster {

const char *F_CLICKS                = "Clicks";
const char *F_COMMERCIAL            = "Commercial";
const char *F_COMMERCIAL_TOP        = "CommercialTop";
const char *F_HOST                  = "Host";
const char *F_NAME                  = "Name";
const char *F_PATH                  = "Path";
const char *F_REQID                 = "ReqId";
//const char *F_SHARE               = "Share";
const char *F_CHATS                 = "Chats";
const char *F_SKILLS                = "Skills";
const char *F_DIALOGS               = "Dialogs";
const char *F_MASCOT_FEATURES       = "Features";
const char *F_MASCOT_KEY            = "Key";
const char *F_PERIOD                = "Period";
const char *F_SPRAV                 = "Sprav";
const char *F_SOURCE                = "Source";
const char *F_TABLE_NO              = "TableNo";
const char *F_TOP                   = "Top";
const char *F_TURBO                 = "Turbo";
const char *F_TURBO_PAGES           = "TurboPages";
const char *F_TURBO_CLICKS          = "TurboClicks";
const char *F_USER_ID               = "UserId";
const char *F_WEBMASTER             = "Webmaster";
const char *F_WEBMASTER_VISITORS    = "WebmasterVisitors";
const char *F_YAMR_KEY              = "key";
const char *F_YAMR_SUBKEY           = "subkey";
const char *F_YAMR_VALUE            = "value";

const ui64 PATH_HASH_TURBO_ENABLED = Max<ui64>();

struct TOwners {
    TOwners() {
        OwnerCanonizer.LoadTrueOwners();
        OwnerCanonizer.LoadSerpCategOwners();
    }

    TString GetOwner(const TString &url) const {
        return TString{OwnerCanonizer.GetUrlOwner(url)};
    }

    static const TOwners& CInstance() {
        return *Singleton<TOwners>();
    }

public:
    TOwnerCanonizer OwnerCanonizer;
};

struct TQueriesMapper : public NYT::IMapper<NYT::TTableReader<NYT::TNode>, NYT::TTableWriter<NYT::TNode>> {
    enum ESource {
        EHost,
        EOwner,
    };

public:
    void Start(TWriter* /*writer*/) override {
        TOwners::CInstance();
    }

    void Do(TReader *input, TWriter *output) override {
        THashMap<TString, size_t> hostCounter;
        THashMap<TString, size_t> ownerCounter;

        for (; input->IsValid(); input->Next()) {
            const NYT::TNode &row = input->GetRow();
            const size_t clicks = row["Clicks"].AsUint64();
            const TString &host = row["Host"].AsString();
            if (clicks > 0) {
                const TString owner = TOwners::CInstance().GetOwner(host);
                hostCounter[host] += clicks;
                ownerCounter[owner] += clicks;
            }
        }

        for (const auto &obj : hostCounter) {
            if (obj.second > 0) {
                output->AddRow(NYT::TNode()
                    (F_NAME, obj.first)
                    (F_SOURCE, TQueriesMapper::EHost)
                    (F_CLICKS, obj.second)
                );
            }
        }

        for (const auto &obj : ownerCounter) {
            if (obj.second > 0) {
                output->AddRow(NYT::TNode()
                    (F_NAME, obj.first)
                    (F_SOURCE, TQueriesMapper::EOwner)
                    (F_CLICKS, obj.second)
                );
            }
        }
    }
};

REGISTER_MAPPER(TQueriesMapper)

//ReduceBy F_NAME, F_SOURCE
struct TQueriesCombiner : public NYT::IReducer<NYT::TTableReader<NYT::TNode>, NYT::TTableWriter<NYT::TNode>> {
    void Do(TReader *input, TWriter *output) override {
        size_t clicks = 0;
        const TString name = input->GetRow()[F_NAME].AsString();
        const int source = input->GetRow()[F_SOURCE].AsInt64();
        for (; input->IsValid(); input->Next()) {
            clicks += input->GetRow()[F_CLICKS].AsUint64();
        }

        output->AddRow(NYT::TNode()
            (F_NAME, name)
            (F_CLICKS, clicks)
            (F_SOURCE, source)
        );
    }
};

REGISTER_REDUCER(TQueriesCombiner)

//ReduceBy F_NAME, F_SOURCE
struct TQueriesReducer : public NYT::IReducer<NYT::TTableReader<NYT::TNode>, NYT::TTableWriter<NYT::TNode>> {
    void Do(TReader *input, TWriter *output) override {
        const ui32 TABLENO_OUTPUT_HOSTS = 0;
        const ui32 TABLENO_OUTPUT_OWNERS = 1;
        size_t clicks = 0;
        const TString name = input->GetRow()[F_NAME].AsString();
        const TQueriesMapper::ESource source = static_cast<TQueriesMapper::ESource>(input->GetRow()[F_SOURCE].AsInt64());
        for (; input->IsValid(); input->Next()) {
            clicks += input->GetRow()[F_CLICKS].AsUint64();
        }

        ui32 outputTableNo;
        switch(source) {
        case TQueriesMapper::EHost:
            outputTableNo = TABLENO_OUTPUT_HOSTS;
            break;
        case TQueriesMapper::EOwner:
            outputTableNo = TABLENO_OUTPUT_OWNERS;
            break;
        }

        output->AddRow(NYT::TNode()
            (F_NAME, name)
            (F_CLICKS, clicks),
            outputTableNo
        );
    }
};

REGISTER_REDUCER(TQueriesReducer)

struct TTurboMapper : public NYT::IMapper<NYT::TTableReader<NYT::TNode>, NYT::TTableWriter<NYT::TNode>> {
    Y_SAVELOAD_JOB(OutputTableConfig)

public:
    TTurboMapper() = default;
    TTurboMapper(const THashMap<ui32, std::pair<time_t, time_t>> &outputTableConfig)
        : OutputTableConfig(outputTableConfig)
    {
    }

    void Do(TReader *input, TWriter *output) override {
        TDeque<THashMap<TString, THashSet<ui64>>> dst;
        dst.resize(OutputTableConfig.size());

        for (; input->IsValid(); input->Next()) {
            const NYT::TNode &row = input->GetRow();
            const time_t timestamp = FromString<time_t>(TStringBuf(row["hash"].AsString()).Before('#'));

            const bool turboEnabled = !NYTUtils::IsNodeNull(row["feed"]) && !row["feed"].AsString().empty();
            const bool turboPage = !NYTUtils::IsNodeNull(row["document"]) && !NYTUtils::IsNodeNull(row["saas_key"]) && row["action"].AsString() == "persist";

            if (!turboEnabled && !turboPage) {
                continue;
            }

            for (const auto &obj : OutputTableConfig) {
                const time_t rangeBegin = obj.second.first;
                const time_t rangeEnd = obj.second.second;
                if (timestamp >= rangeBegin && timestamp < rangeEnd) {
                    TString host, path;
                    if (turboPage) {
                        NUtils::SplitUrl(row["document"].AsString(), host, path);
                        dst[obj.first][host].insert(FnvHash<ui64>(path.data(), path.size()));
                    }
                    if (turboEnabled) {
                        NUtils::SplitUrl(row["feed"].AsString(), host, path);
                        dst[obj.first][host].insert(PATH_HASH_TURBO_ENABLED);
                    }
                    break;
                }
            }
        }

        for (size_t tableNo = 0; tableNo < dst.size(); tableNo++) {
            for (const auto &objHost : dst[tableNo]) {
                const TString &host = objHost.first;
                for (const auto &pathHash : objHost.second) {
                    output->AddRow(NYT::TNode()
                        (F_NAME, host)
                        (F_PATH, pathHash)
                        (F_TABLE_NO, tableNo)
                    );
                }
            }
        }
    }

public:
    THashMap<ui32, std::pair<time_t, time_t>> OutputTableConfig;
};

REGISTER_MAPPER(TTurboMapper)

//ReduceBy F_NAME, F_TABLE_NO
//SortBy F_NAME, F_TABLE_NO, F_PATH
struct TTurboReducer : public NYT::IReducer<NYT::TTableReader<NYT::TNode>, NYT::TTableWriter<NYT::TNode>> {
    void Do(TReader *input, TWriter *output) override {
        const TString name = input->GetRow()[F_NAME].AsString();
        const ui32 tableNo = input->GetRow()[F_TABLE_NO].AsUint64();

        ui64 prevPath = 0;
        size_t paths = 0;
        for (; input->IsValid(); input->Next()) {
            const NYT::TNode &row = input->GetRow();
            const ui64 path = row[F_PATH].AsUint64();
            if (path == PATH_HASH_TURBO_ENABLED) {
                continue;
            }

            if (prevPath != path) {
                prevPath = path;
                paths++;
            }
        }

        output->AddRow(NYT::TNode()
            (F_NAME, name)
            (F_TURBO_PAGES, paths),
            tableNo
        );
    }
};

REGISTER_REDUCER(TTurboReducer)

struct TTurboGetOwnersMapper : public NYT::IMapper<NYT::TTableReader<NYT::TNode>, NYT::TTableWriter<NYT::TNode>> {
    void Start(TWriter *) override {
        TOwners::CInstance();
    }

    void Do(TReader *input, TWriter *output) override {
        for (; input->IsValid(); input->Next()) {
            const NYT::TNode &row = input->GetRow();
            output->AddRow(NYT::TNode()
                (F_NAME, TOwners::CInstance().GetOwner(row[F_NAME].AsString()))
                (F_TURBO_PAGES, row[F_TURBO_PAGES])
            );
        }
    }
};

REGISTER_MAPPER(TTurboGetOwnersMapper)

//ReduceBy F_NAME
struct TTurboGetOwnersReducer : public NYT::IReducer<NYT::TTableReader<NYT::TNode>, NYT::TTableWriter<NYT::TNode>> {
    void Do(TReader *input, TWriter *output) override {
        const TString name = input->GetRow()[F_NAME].AsString();
        size_t turboPages = 0;
        for (; input->IsValid(); input->Next()) {
            const NYT::TNode &row = input->GetRow();
            turboPages += row[F_TURBO_PAGES].AsUint64();
        }
        output->AddRow(NYT::TNode()
            (F_NAME, name)
            (F_TURBO_PAGES, turboPages)
        );
    }
};

REGISTER_REDUCER(TTurboGetOwnersReducer)

//ReduceBy F_YAMR_KEY
//SortBy F_YAMR_KEY, F_YAMR_SUBKEY
struct TParseSearchLogsReducer : public NYT::IReducer<NYT::TTableReader<NYT::TYaMRRow>, NYT::TTableWriter<NYT::TNode>> {
    void Start(TWriter */*writer*/) override {
        PRules.Reset(new TStraightForwardParsingRules);
    }

    void Do(TReader *input, TWriter *output) override {
        const NYT::TYaMRRow &row = input->GetRow();
        const TString id = TString{row.Key};

        for (; input->IsValid(); input->Next()) {
            const TActionItem* action = nullptr;
            try {
                action = PRules->ParseMRData(input->GetRow().Key, input->GetRow().SubKey, input->GetRow().Value);
            } catch (yexception& e) {
                Cerr << e.what() << Endl;
            }

            if (!action) {
                continue;
            }

            if (action->IsA(AT_YANDEX_TURBO_CLICK)) {
                const TYandexTurboClickItem* clickItem = dynamic_cast<const TYandexTurboClickItem*>(action);

                if (clickItem->HasQuery()) {
                    TString host, path;
                    NUtils::SplitUrl(clickItem->GetQuery(), host, path);
                    output->AddRow(NYT::TNode()
                        (F_HOST, host)
                        (F_PATH, path)
                        (F_REQID, clickItem->GetReqID())
                    );
                }
            }
        }
    }

public:
    THolder<TParsingRules> PRules;
};

REGISTER_REDUCER(TParseSearchLogsReducer);

//ReduceBy F_HOST, F_PATH, F_REQID
struct TTurboClicksS1Reducer : public NYT::IReducer<NYT::TTableReader<NYT::TNode>, NYT::TTableWriter<NYT::TNode>> {
    void Do(TReader *input, TWriter *output) override {
        size_t retries = 0;
        const TString host = input->GetRow()[F_HOST].AsString();
        const TString path = input->GetRow()[F_PATH].AsString();
        if (!host.Contains(".")) {
            return;
        }
        for (; input->IsValid(); input->Next()) {
            retries++;
        }
        output->AddRow(NYT::TNode()
            (F_NAME, host)
            (F_PATH, path)
            ("Retries", retries)
        );
    }
};

REGISTER_REDUCER(TTurboClicksS1Reducer)

//ReduceBy F_NAME
//SortBy F_NAME, F_PATH
struct TTurboClicksS2Reducer : public NYT::IReducer<NYT::TTableReader<NYT::TNode>, NYT::TTableWriter<NYT::TNode>> {
    void Do(TReader *input, TWriter *output) override {
        const TString name = input->GetRow()[F_NAME].AsString();
        TString prevPath = 0;
        size_t pages = 0;
        size_t clicks = 0;
        for (; input->IsValid(); input->Next()) {
            const NYT::TNode &row = input->GetRow();
            const TString &path = row[F_PATH].AsString();
            if (prevPath != path) {
                prevPath = path;
                pages++;
            }
            clicks++;
        }
        output->AddRow(NYT::TNode()
            (F_NAME, name)
            (F_TURBO_PAGES, pages)
            (F_CLICKS, clicks)
            ("AvgClicks", static_cast<double>(clicks) / static_cast<double>(pages))
        );
    }
};

REGISTER_REDUCER(TTurboClicksS2Reducer)

struct TTurboClicksGetOwnersMapper : public NYT::IMapper<NYT::TTableReader<NYT::TNode>, NYT::TTableWriter<NYT::TNode>> {
    void Start(TWriter *) override {
        TOwners::CInstance();
    }

    void Do(TReader *input, TWriter *output) override {
        for (; input->IsValid(); input->Next()) {
            const NYT::TNode &row = input->GetRow();
            output->AddRow(NYT::TNode()
                (F_NAME, TOwners::CInstance().GetOwner(row[F_NAME].AsString()))
                (F_TURBO_PAGES, row[F_TURBO_PAGES])
                (F_CLICKS, row[F_CLICKS])
            );
        }
    }
};

REGISTER_MAPPER(TTurboClicksGetOwnersMapper)

//ReduceBy F_NAME
struct TTurboClicksGetOwnersReducer : public NYT::IReducer<NYT::TTableReader<NYT::TNode>, NYT::TTableWriter<NYT::TNode>> {
    void Do(TReader *input, TWriter *output) override {
        const TString name = input->GetRow()[F_NAME].AsString();
        size_t turboPages = 0;
        size_t clicks = 0;
        for (; input->IsValid(); input->Next()) {
            const NYT::TNode &row = input->GetRow();
            turboPages += row[F_TURBO_PAGES].AsUint64();
            clicks += row[F_CLICKS].AsUint64();
        }
        output->AddRow(NYT::TNode()
            (F_NAME, name)
            (F_TURBO_PAGES, turboPages)
            (F_CLICKS, clicks)
            ("AvgClicks", static_cast<double>(clicks) / static_cast<double>(turboPages))
        );
    }
};

REGISTER_REDUCER(TTurboClicksGetOwnersReducer)

struct TMascotCMMapper : public NYT::IMapper<NYT::TTableReader<NYT::TNode>, NYT::TTableWriter<NYT::TNode>> {
    void Do(TReader *input, TWriter *output) override {
        for (; input->IsValid(); input->Next()) {
            const NYT::TNode &row = input->GetRow();

            NMascot::TFeatures features;
            Y_PROTOBUF_SUPPRESS_NODISCARD features.ParseFromString(row[F_MASCOT_FEATURES].AsString());
            if (features.GetVisits() == 0) {
                continue;
            }

            if (features.GetQueriesAvgCM2() > 0.3 || features.GetQueriesCommRatioGt04() > 0.3) { //https://st.yandex-team.ru/WMC-5777#1528534868000
                output->AddRow(NYT::TNode()
                    (F_NAME, row[F_MASCOT_KEY])
                    ("Visits", features.GetVisits())
                    ("QueriesAvgCM2", features.GetQueriesAvgCM2())
                    ("QueriesCommRatioGt04", features.GetQueriesCommRatioGt04())
                );
            }
        }
    }
};

REGISTER_MAPPER(TMascotCMMapper)

struct TWebmasterVerifiedHostsMapper : public NYT::IMapper<NYT::TTableReader<NYT::TNode>, NYT::TTableWriter<NYT::TNode>> {
    void Start(TWriter *) override {
        TOwners::CInstance();
    }

    void Do(TReader *input, TWriter *output) override {
        for (; input->IsValid(); input->Next()) {
            const NYT::TNode &row = input->GetRow();
            output->AddRow(NYT::TNode()
                (F_NAME, TOwners::CInstance().GetOwner(row[F_YAMR_KEY].AsString()))
            );
        }
    }
};

REGISTER_MAPPER(TWebmasterVerifiedHostsMapper)

//ReduceBy F_NAME
struct TWebmasterVerifiedHostsReducer : public NYT::IReducer<NYT::TTableReader<NYT::TNode>, NYT::TTableWriter<NYT::TNode>> {
    void Do(TReader *input, TWriter *output) override {
        output->AddRow(NYT::TNode()
            (F_NAME, input->GetRow()[F_NAME])
        );
    }
};

REGISTER_REDUCER(TWebmasterVerifiedHostsReducer)

struct TWebmasterFrontLogRecord {
    TWebmasterFrontLogRecord(const TString &message) {
        //static TRegularExpression regex("^\\[pid:\\d+\\] Resolved request <(.+)> \\[in .+ ms\\] for (.+)( ?{?.*}?)?$");
        //static TRegularExpression regex("^\\[pid:\\d+?\\] Resolved request <(.+?)> \\[in .+? ms\\].*? for (.+?)");
        static TRegularExpression regex("^\\[pid:\\d+\\] Resolved request <(.+?)> \\[in .+?\\~(\\d+) ms\\].*? for (.+)");

        TVector<TString> hits;
        const size_t hitsCount = regex.GetMatches(message, hits);

        if (hitsCount == 3) {
            //const TString &action = hits[0];
            TimeMs = FromString<size_t>(hits[1]);
            const TString &rest = hits[2];
            Url = TString{TStringBuf(rest).Before(' ')};

            if (GetHttpPrefixSize(Url) == 0) {
                Url = TString::Join("http://", Url);
            }

            NUri::TUri parsedUrl;
            if (parsedUrl.Parse(Url) != NUri::TState::ParsedOK) {
                ythrow yexception() << "unable to parse url: " << Url << Endl;
            }

            Host = TString{parsedUrl.GetField(NUri::TField::FieldHost)};
            Query = parsedUrl.GetField(NUri::TField::FieldQuery);
            Method = parsedUrl.GetField(NUri::TField::FieldPath);
            Params = TCgiParameters(Query);
        } else {
            ythrow yexception() << "unknown message: " << message;
        }
    }

public:
    TString Url;
    TString Host;
    TString Method;
    TString Query;
    size_t TimeMs;
    TCgiParameters Params;
};

struct TExtractWebmasterFrontLogsMapper : public NYT::IMapper<NYT::TTableReader<NYT::TNode>, NYT::TTableWriter<NYT::TNode>> {
public:
    void Start(TWriter *) override {
        TOwners::CInstance();
    }

    void Do(TReader *input, TWriter *output) override {
        THashMap<TString, THashSet<size_t>> visitsHosts;
        THashMap<TString, THashSet<size_t>> visitsOwners;
        for (; input->IsValid(); input->Next()) {
            using namespace NYTUtils;
            const NYT::TNode &row = input->GetRow();
            const TString message = FromNodeOrDefault<TString>(row["message"], "");

            try {
                if (FromNodeOrDefault<TString>(row["qloud_project"], "") == "webmaster"
                    && FromNodeOrDefault<TString>(row["qloud_application"], "") == "webmaster-www"
                    && FromNodeOrDefault<TString>(row["qloud_environment"], "") == "production"
                    && FromNodeOrDefault<TString>(row["loggerName"], "") == "stdout"
                    && message.Contains("Resolved request <")
                ) {
                    TWebmasterFrontLogRecord record(message);
                    if (record.Host != "webmaster3-viewer.search.yandex.net") {
                        continue;
                    }

                    size_t userId = 0;
                    TString hostId;
                    for (const auto &obj : record.Params) {
                        if (obj.first == "userId") {
                            userId = FromString<size_t>(obj.second);
                        } else if (obj.first == "hostId") {
                            long tmpHostId = 0;
                            if (!obj.second.empty() && !TryFromString(obj.second, tmpHostId)) {
                                hostId = TWebmasterHostId::FromHostId(obj.second).ToHostName();
                            }
                        }
                    }

                    if (!hostId.empty() && userId != 0) {
                        TString asciiHostId;
                        if (NUtils::IDNHostToAscii(hostId, asciiHostId)) {
                            visitsHosts[asciiHostId].insert(userId);
                            visitsOwners[TOwners::CInstance().GetOwner(asciiHostId)].insert(userId);
                        }
                    }
                }
            } catch (yexception &e) {
                NYT::TNode dstRow = row;
                Cerr << e.what() << Endl;
            }
        }

        for (const auto &obj : visitsHosts) {
            const TString &hostId = obj.first;
            for (size_t userId : obj.second) {
                output->AddRow(NYT::TNode()
                    (F_NAME, hostId)
                    (F_USER_ID, userId)
                    (F_TABLE_NO, "Hosts")
                );
            }
        }

        for (const auto &obj : visitsOwners) {
            const TString &hostId = obj.first;
            for (size_t userId : obj.second) {
                output->AddRow(NYT::TNode()
                    (F_NAME, hostId)
                    (F_USER_ID, userId)
                    (F_TABLE_NO, "Owners")
                );
            }
        }
    }
};

REGISTER_MAPPER(TExtractWebmasterFrontLogsMapper)

//ReduceBy F_NAME, F_USER_ID, F_TABLE_NO
struct TExtractWebmasterFrontLogsReducer : public NYT::IReducer<NYT::TTableReader<NYT::TNode>, NYT::TTableWriter<NYT::TNode>> {
public:
    void Do(TReader *input, TWriter *output) override {
        const ui32 TABLENO_HOSTS    = 0;
        const ui32 TABLENO_OWNERS   = 1;
        const NYT::TNode &row = input->GetRow();
        const ui32 tableNo = row[F_TABLE_NO].AsString() == "Hosts" ? TABLENO_HOSTS : TABLENO_OWNERS;
        output->AddRow(NYT::TNode()
            (F_NAME, row[F_NAME])
            (F_USER_ID, row[F_USER_ID]),
            tableNo
        );
    }
};

REGISTER_REDUCER(TExtractWebmasterFrontLogsReducer)

/*
void UpdateShare(NYT::IClientBasePtr client, const TString &src, const TString &dst) {
    TDeque<std::pair<size_t, TString>> counters;
    auto reader = client->CreateTableReader<NYT::TNode>(src);
    size_t total = 0;
    for (; reader->IsValid(); reader->Next()) {
        const NYT::TNode &row = reader->GetRow();
        const size_t clicks = row[F_CLICKS].AsUint64();
        const TString name = row[F_NAME].AsString();
        if (!name.empty()) {
            counters.push_back(std::make_pair(clicks, name));
            total += clicks;
        }
    }

    std::sort(counters.begin(), counters.end(), [](const std::pair<TString, size_t> &lhs, const std::pair<TString, size_t> &rhs) -> bool {
        return rhs.second < lhs.second;
    });

    auto writer = client->CreateTableWriter<NYT::TNode>(dst);
    for (const auto &obj : counters) {
        writer->AddRow(NYT::TNode()
            (F_NAME, obj.first)
            (F_CLICKS, obj.second)
            (F_SHARE, static_cast<double>(obj.second) / static_cast<double>(total))
        );
    }
    writer->Finish();
}
*/
struct TMonthRangeConfig {
    static TMonthRangeConfig Today() {
        return Next(TDate::Today());
    }

    static TMonthRangeConfig Next(const TDate &date) {
        TMonthRangeConfig obj;
        obj.RangeStart = TDate(date.GetYear(), date.GetMonth(), 1);
        obj.RangeEnd = GetNextRangeDate(obj.RangeStart);
        obj.Current = obj.In(TDate::Today());
        return obj;
    }

    static TMonthRangeConfig Next(const TMonthRangeConfig &rc) {
        return Next(rc.RangeEnd);
    }

    static TMonthRangeConfig Prev(const TDate &date) {
        TMonthRangeConfig obj;
        obj.RangeEnd = TDate(date.GetYear(), date.GetMonth(), 1);
        obj.RangeStart = GetPrevRangeDate(obj.RangeEnd);
        obj.Current = obj.In(TDate::Today());
        return obj;
    }

    static TMonthRangeConfig Prev(const TMonthRangeConfig &rc) {
        return Prev(rc.RangeStart);
    }

    static TDate GetNextRangeDate(const TDate &date) {
        unsigned month = date.GetMonth();
        unsigned year = date.GetYear();
        month++;
        if (month == 13) {
            month = 1;
            year++;
        }
        return TDate(year, month, 1);
    }

    static TDate GetPrevRangeDate(const TDate &date) {
        unsigned month = date.GetMonth();
        unsigned year = date.GetYear();
        month--;
        if (month == 0) {
            month = 12;
            year--;
        }
        return TDate(year, month, 1);
    }

    void GetDays(TDeque<TDate> &days) const {
        TDate date = RangeStart;
        while (date < RangeEnd) {
            days.push_back(date);
            date += 1;
        }
    }

    unsigned GetDaysCount() const {
        return (RangeEnd.GetStart() - RangeStart.GetStart()) / SECONDS_IN_DAY;
    }

    bool In(const TDate &date) const {
        return date >= RangeStart && date < RangeEnd;
    }

    bool operator<(const TMonthRangeConfig &rhs) const {
        return RangeStart < rhs.RangeStart;
    }

    TString Name() const {
        return RangeStart.ToStroka() + "_" + RangeEnd.ToStroka();
    }

    friend IOutputStream& operator<<(IOutputStream& left, const TMonthRangeConfig& right);

public:
    TDate RangeStart;
    TDate RangeEnd;
    bool Current = false;
};

struct TWeekRangeConfig {
    static TWeekRangeConfig Today() {
        return Next(TDate::Today());
    }

    static TWeekRangeConfig Next(const TDate &date) {
        TWeekRangeConfig obj;
        obj.RangeStart = GetStartOfRange(date);
        obj.RangeEnd = GetNextRangeDate(obj.RangeStart);
        obj.Current = obj.In(TDate::Today());
        return obj;
    }

    static TWeekRangeConfig Next(const TWeekRangeConfig &rc) {
        return Next(rc.RangeEnd);
    }

    static TWeekRangeConfig Prev(const TDate &date) {
        TWeekRangeConfig obj;
        obj.RangeEnd = GetStartOfRange(date);
        obj.RangeStart = GetPrevRangeDate(obj.RangeEnd);
        obj.Current = obj.In(TDate::Today());
        return obj;
    }

    static TWeekRangeConfig Prev(const TWeekRangeConfig &rc) {
        return Prev(rc.RangeStart);
    }

    static TDate GetStartOfRange(const TDate &date) {
        const static int WEEK_START_OFFSET_DAYS = 6;
        int weekDay = (GetWeekDay(date) + WEEK_START_OFFSET_DAYS) % 7;
        return TDate(date.GetStart()) - weekDay;
    }

    static unsigned GetWeekDay(const TDate &date) {
        const time_t ts = date.GetStart();
        struct tm *ti = localtime(&ts);
        return ti->tm_wday;
    }

    static TDate GetNextRangeDate(const TDate &date) {
        return date + 7;
    }

    static TDate GetPrevRangeDate(const TDate &date) {
        return date - 7;
    }

    void GetDays(TDeque<TDate> &days) const {
        TDate date = RangeStart;
        while (date < RangeEnd) {
            days.push_back(date);
            date += 1;
        }
    }

    unsigned GetDaysCount() const {
        return (RangeEnd.GetStart() - RangeStart.GetStart()) / SECONDS_IN_DAY;
    }

    bool In(const TDate &date) const {
        return date >= RangeStart && date < RangeEnd;
    }

    bool operator<(const TWeekRangeConfig &rhs) const {
        return RangeStart < rhs.RangeStart;
    }

    TString Name() const {
        return RangeStart.ToStroka() + "_" + RangeEnd.ToStroka();
    }

    friend IOutputStream& operator<<(IOutputStream& left, const TWeekRangeConfig& right);

public:
    TDate RangeStart;
    TDate RangeEnd;
    bool Current = false;
};

//using TRangeConfig = TMonthRangeConfig;
using TRangeConfig = TWeekRangeConfig;

inline IOutputStream& operator<<(IOutputStream& left, const TRangeConfig& right) {
    return left << right.Name();
}

struct TTablePack {
    using Ptr = TSimpleSharedPtr<TTablePack>;

    struct TOutputTables {
    public:
        TString Chats;
        TString Clicks;
        TString Dialogs;
        TString Mascot;
        TString Report;
        TString Skills;
        TString Sprav;
        TString Turbo;
        TString Webmaster;
        TString WebmasterVisits;
    };

public:
    bool IsChatsTablesReady(NYT::IClientBasePtr client) const {
        return client->Exists(OutputOwners.Chats);
    }

    bool IsClicksTableSetComplete(const TRangeConfig &range) const {
        return range.GetDaysCount() == InputClicks.size();
    }

    bool IsClicksTablesReady(NYT::IClientBasePtr client) const {
        return client->Exists(OutputHosts.Clicks) && client->Exists(OutputOwners.Clicks);
    }

    bool IsDialogsTablesReady(NYT::IClientBasePtr client) const {
        return client->Exists(OutputOwners.Dialogs);
    }

    bool IsMascotTablesReady(NYT::IClientBasePtr client) const {
        return client->Exists(OutputOwners.Mascot);
    }

    bool IsSerpLogsTableSetComplete(const TRangeConfig &range) const {
        return range.GetDaysCount() == InputSerpLogs.size();
    }

    bool IsSkillsTablesReady(NYT::IClientBasePtr client) const {
        return client->Exists(OutputOwners.Skills);
    }

    bool IsSpravTablesReady(NYT::IClientBasePtr client) const {
        return client->Exists(OutputOwners.Sprav);
    }

    bool IsTurboTablesReady(NYT::IClientBasePtr client) const {
        return /*client->Exists(OutputHosts.Turbo) &&*/ client->Exists(OutputOwners.Turbo);
    }

    bool IsWebmasterTablesReady(NYT::IClientBasePtr client) const {
        return client->Exists(OutputOwners.Webmaster);
    }

    bool IsWebmasterFrontLogsTableSetComplete(const TRangeConfig &range) const {
        return range.GetDaysCount() == InputWebmasterFrontLogs.size();
    }

    bool IsWebmasterVisitsTablesReady(NYT::IClientBasePtr client) const {
        return client->Exists(OutputOwners.WebmasterVisits);
    }

public:
    TDeque<TString> InputSerpLogs;
    TDeque<TString> InputClicks;
    TDeque<TString> InputWebmasterFrontLogs;

public:
    TOutputTables OutputHosts;
    TOutputTables OutputOwners;
};

struct TCounters {
public:
    size_t Clicks           = 0;
    //double Share            = 0;
    bool Turbo              = false;
    bool Sprav              = false;
    bool Chats              = false;
    bool Commercial         = false;
    bool Dialogs            = false;
    bool Skills             = false;
    size_t TurboClicks      = 0;
    size_t TurboPages       = 0;
    size_t Top              = Max<size_t>();
    size_t CommercialTop    = Max<size_t>();
    bool Webmaster          = false;
    size_t WebmasterVisits  = 0;
};

struct TClients {
    NYT::IClientPtr ChatNSkills;
    //NYT::IClientPtr Dialogs;
    NYT::IClientPtr Mascot;
    NYT::IClientPtr Queries;
    NYT::IClientPtr Sources;
    NYT::IClientPtr Sprav;
    NYT::IClientPtr Turbo;
    NYT::IClientPtr Webmaster;
    NYT::IClientPtr WebmasterVisits;
};

void LoadClicksTablesList(NYT::IClientBasePtr client, TMap<TRangeConfig, TTablePack::Ptr> &ranges) {
    const char *FORMAT = "%Y-%m-%d";

    TDeque<NYTUtils::TTableInfo> tablesList;
    if (!NYTUtils::GetTableList(client, TCommonYTConfigSQ::CInstance().TABLE_PARSED_USER_SESSIONS_STATS_DAILY_ROOT, tablesList, Max<int>())) {
        ythrow yexception() << "No input tables found";
    }

    for (const NYTUtils::TTableInfo& table : tablesList) {
        const TDate tableDate(NYTUtils::GetTableName(table.Name), FORMAT);
        for (auto &rcObj : ranges) {
            if (rcObj.first.In(tableDate)) {
                rcObj.second->InputClicks.push_back(table.Name);
                break;
            }
        }
    }
}

void LoadSerpLogsTablesList(NYT::IClientBasePtr client, TMap<TRangeConfig, TTablePack::Ptr> &ranges) {
    const char *FORMAT = "%Y-%m-%d";
    for (auto &rcObj : ranges) {
        const TRangeConfig &range = rcObj.first;
        const TTablePack::Ptr &tablePack = rcObj.second;
        TDeque<TDate> days;
        range.GetDays(days);
        for (const TDate &day : days) {
            const TString inputTable = "//user_sessions/pub/search/daily/" + day.ToStroka(FORMAT) + "/clean";
            if (client->Exists(inputTable)) {
                tablePack->InputSerpLogs.push_back(inputTable);
            }
        }
    }
}

void LoadWebmasterFrontLogsTablesList(NYT::IClientBasePtr client, TMap<TRangeConfig, TTablePack::Ptr> &ranges) {
    const char *FORMAT = "%Y-%m-%d";
    for (auto &rcObj : ranges) {
        const TRangeConfig &range = rcObj.first;
        const TTablePack::Ptr &tablePack = rcObj.second;
        TDeque<TDate> days;
        range.GetDays(days);
        for (const TDate &day : days) {
            const TString inputTable = NYTUtils::JoinPath("//home/logfeller/logs/qloud-runtime-log/1d", day.ToStroka(FORMAT));
            if (client->Exists(inputTable)) {
                tablePack->InputWebmasterFrontLogs.push_back(inputTable);
            }
        }
    }
}

void BuildClicksSource(NYT::IClientBasePtr tx, const TTablePack::Ptr &tablePack) {
    TOpRunner runner(tx);

    for (const TString &table : tablePack->InputClicks) {
        runner.InputNode(table);
    }

    runner
        .OutputNode(NYT::TRichYPath(tablePack->OutputHosts.Clicks))
        .OutputNode(NYT::TRichYPath(tablePack->OutputOwners.Clicks))
        .ReduceBy(F_NAME, F_SOURCE)
        .MapReduce(new TQueriesMapper, new TQueriesReducer)
        .SortBy(F_NAME)
        .Sort(tablePack->OutputHosts.Clicks, ASYNC_CTX0)
        .SortBy(F_NAME)
        .Sort(tablePack->OutputOwners.Clicks, ASYNC_CTX0)
        .Wait(ASYNC_CTX0)
    ;

    //UpdateShare(tx, tablePack->OutputHosts.Clicks, tablePack->OutputHosts.Clicks);
    //UpdateShare(tx, tablePack->OutputOwners.Clicks, tablePack->OutputOwners.Clicks);
}

void BuildClicksSource(NYT::IClientBasePtr clientQueries, const TMap<TRangeConfig, TTablePack::Ptr> &ranges) {
    THolder<IThreadPool> processQueue(CreateThreadPool(4));
    for (const auto &rcObj : ranges) {
        const TRangeConfig &range = rcObj.first;
        const TTablePack::Ptr &tablePack = rcObj.second;

        if (!tablePack->IsClicksTableSetComplete(range)) {
            if (!range.Current) {
                LOG_WARN("range %s, serp clicks is incomplete: %lu / %u", range.Name().data(), tablePack->InputClicks.size(), range.GetDaysCount());
            }
            continue;
        }

        if (tablePack->IsClicksTablesReady(clientQueries)) {
            continue;
        }

        processQueue->SafeAddFunc([=, &clientQueries]() {
            try {
                NYT::ITransactionPtr tx = clientQueries->StartTransaction();
                BuildClicksSource(tx, tablePack);
                tx->Commit();
            } catch(yexception &e) {
                LOG_ERROR("process range %s error: %s", rcObj.first.Name().data(), e.what());
            }
        });
    }
    processQueue->Stop();
}
/*
void BuildSourceTurbo(NYT::IClientBasePtr clientQueries, const TString &tableTurboHistory, const TMap<TRangeConfig, TTablePack::Ptr> &ranges) {
    NYT::ITransactionPtr tx = clientQueries->StartTransaction();
    TOpRunner runner(tx);

    TDeque<TTablePack::Ptr> processed;
    THashMap<ui32, std::pair<time_t, time_t>> outputTableConfig;
    for (auto &rcObj : ranges) {
        const TRangeConfig &range = rcObj.first;
        const TTablePack::Ptr &tablePack = rcObj.second;

        if (tablePack->IsTurboTablesReady(tx)) {
            continue;
        }

        if (range.Current || !tablePack->IsClicksTableSetComplete(range)) {
            continue;
        }

        if (!tablePack->IsClicksTablesReady(clientQueries)) {
            LOG_WARN("range %s, turbo: serp clicks is not ready", range.Name().data());
            continue;
        }

        runner.OutputNode(tablePack->OutputHosts.Turbo);
        outputTableConfig[outputTableConfig.size()] = std::make_pair(range.RangeStart.GetStart(), range.RangeEnd.GetStart());
        processed.push_back(tablePack);
    }

    if (!outputTableConfig.empty()) {
        runner
            .InputNode(tableTurboHistory)
            .ReduceBy(F_NAME, F_TABLE_NO)
            .SortBy(F_NAME, F_TABLE_NO, F_PATH)
            .MapReduce(new TTurboMapper(outputTableConfig), new TTurboReducer)
        ;
    }

    for (TTablePack::Ptr tablePack : processed) {
        runner
            .InputNode(tablePack->OutputHosts.Turbo)
            .OutputNode(tablePack->OutputOwners.Turbo)
            .ReduceBy(F_NAME)
            .MapReduce(new TTurboGetOwnersMapper, new TTurboGetOwnersReducer)
            .SortBy(F_NAME)
            .Sort(tablePack->OutputHosts.Turbo, ASYNC_CTX0)
            .SortBy(F_NAME)
            .Sort(tablePack->OutputOwners.Turbo, ASYNC_CTX0)
            .Wait(ASYNC_CTX0)
        ;
    }

    tx->Commit();
}
*/
void WriteSpravSource(NYT::IClientBasePtr client, const TString &src, const TMap<TRangeConfig, TTablePack::Ptr> &ranges) {
    TSet<TString> owners;
    auto reader = client->CreateTableReader<NYT::TNode>(src);
    for (; reader->IsValid(); reader->Next()) {
        const NYT::TNode &row = reader->GetRow();
        const TString &url = row["url"].AsString();
        if (url.empty()) {
            continue;
        }
        TString host, path, asciiHost;
        NUtils::SplitUrl(url, host, path);
        if (NUtils::IDNHostToAscii(host, asciiHost)) {
            const TString owner = to_lower(TOwners::CInstance().GetOwner(asciiHost));
            owners.insert(owner);
        }
    }

    for (const auto &rcObj : ranges) {
        const TRangeConfig &range = rcObj.first;
        const TTablePack::Ptr &tablePack = rcObj.second;
        if (range.Current || tablePack->IsSpravTablesReady(client)) {
            continue;
        }
        LOG_INFO("source sprav %s", range.Name().data());
        auto writer = client->CreateTableWriter<NYT::TNode>(NYT::TRichYPath(tablePack->OutputOwners.Sprav).SortedBy(F_NAME));
        for (const TString &owner : owners) {
            writer->AddRow(NYT::TNode()
                (F_NAME, owner)
            );
        }
        writer->Finish();
        LOG_INFO("source sprav %s - done", range.Name().data());
    }
}
/*
void WriteDialogsSource(NYT::IClientBasePtr clientDialogs, NYT::IClientBasePtr clientSources, const TString &src, const TMap<TRangeConfig, TTablePack::Ptr> &ranges) {
    TSet<TString> owners;
    auto reader = clientDialogs->CreateTableReader<NYT::TNode>(src);
    for (; reader->IsValid(); reader->Next()) {
        const NYT::TNode &row = reader->GetRow();
        const TString &url = row["Dialog"]["brand_website"].AsString();
        if (url.empty()) {
            continue;
        }
        TString host, path, asciiHost;
        NUtils::SplitUrl(url, host, path);
        if (NUtils::IDNHostToAscii(host, asciiHost)) {
            const TString owner = TOwners::CInstance().GetOwner(asciiHost);
            owners.insert(owner);
        }
    }

    for (const auto &rcObj : ranges) {
        const TRangeConfig &range = rcObj.first;
        const TTablePack::Ptr &tablePack = rcObj.second;
        if (tablePack->IsDialogsTablesReady(clientSources)) {
            continue;
        }
        LOG_INFO("source dialogs %s", range.Name().data());
        auto writer = clientSources->CreateTableWriter<NYT::TNode>(NYT::TRichYPath(tablePack->OutputOwners.Dialogs).SortedBy(F_NAME));
        for (const TString &owner : owners) {
            writer->AddRow(NYT::TNode()
                (F_NAME, owner)
            );
        }
        writer->Finish();
        LOG_INFO("source dialogs %s - done", range.Name().data());
    }
}
*/
void WriteDialogsChatsAndSkillsSource(NYT::IClientBasePtr client, const TString &src, const TMap<TRangeConfig, TTablePack::Ptr> &ranges) {
    TSet<TString> chatsOwners, skillsOwners, dialogsOwners;
    auto reader = client->CreateTableReader<NYT::TNode>(src);
    for (; reader->IsValid(); reader->Next()) {
        const NYT::TNode &row = reader->GetRow();
        const TString channel = row["channel"].AsString();
        const TString url = NYTUtils::FromNodeOrDefault<TString>(row["brandVerificationWebsite"], "");
        if (!row["onAir"].AsBool() || url.empty()) {
            continue;
        }
        TString host, path, asciiHost;
        NUtils::SplitUrl(url, host, path);
        if (NUtils::IDNHostToAscii(host, asciiHost)) {
            const TString owner = to_lower(TOwners::CInstance().GetOwner(asciiHost));
            if (channel == "aliceSkill") {
                skillsOwners.insert(owner);
            } else if (channel == "organizationChat") {
                chatsOwners.insert(owner);
            }
            dialogsOwners.insert(owner);
        }
    }

    TDeque<TString> moreSkills = { //https://st.yandex-team.ru/WMC-5777#1529007134000
        "kinopoisk.ru",
        "yandex.ru",
        "news.yandex.ru",
        "market.yandex.ru",
        "sports.ru",
        "s7.ru",
        "fss.ru",
        "sony.ru",
        "pecom.ru",
        "radio-t.com",
        "arzamas.academy",
        "totaldict.ru",
        "2ch.hk",
    };

    for (const TString &name : moreSkills) {
        skillsOwners.insert(name);
        dialogsOwners.insert(name);
    }

    for (const auto &rcObj : ranges) {
        const TRangeConfig &range = rcObj.first;
        const TTablePack::Ptr &tablePack = rcObj.second;
        if (range.Current) {
            continue;
        }

        if (!tablePack->IsChatsTablesReady(client)) {
            LOG_INFO("source chats %s", range.Name().data());
            auto writer = client->CreateTableWriter<NYT::TNode>(NYT::TRichYPath(tablePack->OutputOwners.Chats).SortedBy(F_NAME));
            for (const TString &owner : chatsOwners) {
                writer->AddRow(NYT::TNode()
                    (F_NAME, owner)
                );
            }
            writer->Finish();
            LOG_INFO("source chats %s - done", range.Name().data());
        }

        if (!tablePack->IsSkillsTablesReady(client)) {
            LOG_INFO("source skills %s", range.Name().data());
            auto writer = client->CreateTableWriter<NYT::TNode>(NYT::TRichYPath(tablePack->OutputOwners.Skills).SortedBy(F_NAME));
            for (const TString &owner : skillsOwners) {
                writer->AddRow(NYT::TNode()
                    (F_NAME, owner)
                );
            }
            writer->Finish();
            LOG_INFO("source skills - done %s", range.Name().data());
        }

        if (!tablePack->IsDialogsTablesReady(client)) {
            LOG_INFO("source dialogs %s", range.Name().data());
            auto writer = client->CreateTableWriter<NYT::TNode>(NYT::TRichYPath(tablePack->OutputOwners.Dialogs).SortedBy(F_NAME));
            for (const TString &owner : dialogsOwners) {
                writer->AddRow(NYT::TNode()
                    (F_NAME, owner)
                );
            }
            writer->Finish();
            LOG_INFO("source dialogs - done %s", range.Name().data());
        }
    }
}

void BuildMascotCommercialSource(NYT::IClientBasePtr clientMascot, const TString &srcRoot, const TMap<TRangeConfig, TTablePack::Ptr> &ranges) {
    TMap<TRangeConfig, TSet<TString>> mascotRanges;
    TDeque<NYTUtils::TTableInfo> mascotTables;
    NYTUtils::GetTableList(clientMascot, srcRoot, mascotTables);
    for (const NYTUtils::TTableInfo &table : mascotTables) {
        const TDate timestamp = TDate(FromString<time_t>(TStringBuf(table.Name).RAfter('.')));
        for (const auto &obj : ranges) {
            if (!obj.first.Current && obj.first.In(timestamp)) {
                mascotRanges[obj.first].insert(table.Name);
                break;
            }
        }
    }

    THashSet<TString> processed;
    for (const auto &obj : mascotRanges) {
        if (obj.second.empty()) {
            continue;
        }

        NYT::ITransactionPtr tx = clientMascot->StartTransaction();
        const TRangeConfig &range = obj.first;
        const TString &mascotTable = *obj.second.rbegin();
        const TTablePack::Ptr &tablePack = ranges.at(range);
        const TString &outputTable = tablePack->OutputOwners.Mascot;
        if (!tablePack->IsMascotTablesReady(clientMascot)) {
            TOpRunner(tx)
                .InputNode(mascotTable)
                .OutputNode(NYT::TRichYPath(outputTable).SortedBy(F_NAME))
                .EnableOrderedMap()
                .Map(new TMascotCMMapper)
                .SortBy(F_NAME)
                .Sort(outputTable)
            ;
            tx->Commit();
        }
        processed.insert(outputTable);
    }
    /*
    TString prevSource;
    for (const auto &obj : ranges) { //fill gaps
        if (processed.has(obj.second->OutputOwners.Mascot) || obj.second->IsMascotTablesReady(clientMascot)) {
            prevSource = obj.second->OutputOwners.Mascot;
        } else if (!prevSource.empty() && !obj.first.Current) {
            TOpRunner(clientMascot).Copy(prevSource, obj.second->OutputOwners.Mascot);
        }
    }
    */
}

void BuildWebmasterSource(NYT::IClientBasePtr client, const TString &srcRoot, const TMap<TRangeConfig, TTablePack::Ptr> &ranges) {
    TMap<TRangeConfig, TSet<TString>> webmasterRanges;
    TDeque<NYTUtils::TTableInfo> tables;
    NYTUtils::GetTableList(client, srcRoot, tables);

    for (const NYTUtils::TTableInfo &table : tables) {
        if (table.Name.Contains("/ready-") || table.Name.Contains("/transmitted-")) {
            const TDate timestamp = TDate(FromString<time_t>(TStringBuf(table.Name).RAfter('-')));
            for (const auto &obj : ranges) {
                if (!obj.first.Current && obj.first.In(timestamp)) {
                    webmasterRanges[obj.first].insert(table.Name);
                    break;
                }
            }
        }
    }

    THolder<IThreadPool> processQueue(CreateThreadPool(4));
    for (const auto &obj : webmasterRanges) {
        if (obj.second.empty()) {
            continue;
        }

        processQueue->SafeAddFunc([=, &client]() {
            try {
               NYT::ITransactionPtr tx = client->StartTransaction();
               const TRangeConfig &range = obj.first;
               const TString &webmasterTable = *obj.second.rbegin();
               const TTablePack::Ptr &tablePack = ranges.at(range);
               if (!tablePack->IsWebmasterTablesReady(client)) {
                   const TString &outputTable = tablePack->OutputOwners.Webmaster;
                   TOpRunner(tx)
                       .InputNode(webmasterTable)
                       .OutputNode(NYT::TRichYPath(outputTable))
                       .ReduceBy(F_NAME)
                       .MapReduce(new TWebmasterVerifiedHostsMapper, new TWebmasterVerifiedHostsReducer)
                       .SortBy(F_NAME)
                       .Sort(outputTable)
                   ;
                   tx->Commit();
               }
            } catch(yexception &e) {
                LOG_ERROR("process range %s error: %s", obj.first.Name().data(), e.what());
            }
        });
    }

    processQueue->Stop();
}

void BuildWebmasterVisitsSource(NYT::IClientBasePtr client, const TMap<TRangeConfig, TTablePack::Ptr> &ranges) {
    THolder<IThreadPool> processQueue(CreateThreadPool(2));
    for (const auto &rcObj : ranges) {
        const TRangeConfig &range = rcObj.first;
        const TTablePack::Ptr &tablePack = rcObj.second;

        if (!tablePack->IsWebmasterFrontLogsTableSetComplete(range)) {
            if (!range.Current) {
                LOG_WARN("range %s, webmaster front logs is incomplete: %lu / %u", range.Name().data(), tablePack->InputWebmasterFrontLogs.size(), range.GetDaysCount());
            }
            continue;
        }

        if (tablePack->IsWebmasterVisitsTablesReady(client)) {
            continue;
        }

        processQueue->SafeAddFunc([=, &client]() {
            try {
                NYT::ITransactionPtr tx = client->StartTransaction();
                TOpRunner runner(tx);

                for (const TString &table : tablePack->InputWebmasterFrontLogs) {
                    runner.InputNode(table);
                }

                runner
                    .OutputNode(NYT::TRichYPath(tablePack->OutputHosts.WebmasterVisits))
                    .OutputNode(NYT::TRichYPath(tablePack->OutputOwners.WebmasterVisits))
                    .ReduceBy(F_NAME, F_USER_ID, F_TABLE_NO)
                    .MapReduce(new TExtractWebmasterFrontLogsMapper, new TExtractWebmasterFrontLogsReducer)
                    .SortBy(F_NAME)
                    .Sort(tablePack->OutputHosts.WebmasterVisits, ASYNC_CTX0)
                    .SortBy(F_NAME)
                    .Sort(tablePack->OutputOwners.WebmasterVisits, ASYNC_CTX0)
                    .Wait(ASYNC_CTX0)
                ;

                tx->Commit();
            } catch(yexception &e) {
                LOG_ERROR("process range %s error: %s", rcObj.first.Name().data(), e.what());
            }
        });
    }
    processQueue->Stop();
}

void ReadSources(const TClients &clients, const TTablePack::TOutputTables &outputTables, TMap<TString, TCounters> &counters) {
    auto reader = clients.Queries->CreateTableReader<NYT::TNode>(outputTables.Clicks);
    for (; reader->IsValid(); reader->Next()) {
        const NYT::TNode &row = reader->GetRow();
        TCounters &tmp = counters[row[F_NAME].AsString()];
        tmp.Clicks = row[F_CLICKS].AsUint64();
        //tmp.Share = row[F_SHARE].AsDouble();
    }

    reader = clients.Turbo->CreateTableReader<NYT::TNode>(outputTables.Turbo);
    for (; reader->IsValid(); reader->Next()) {
        const NYT::TNode &row = reader->GetRow();
        if (!NYTUtils::IsNodeNull(row["domain"])) {
            const TString &name = row["domain"].AsString();
            counters[name].Turbo = true;
            counters[name].TurboClicks = row["turbo_clicks"].AsUint64();
        }
    }

    reader = clients.Sprav->CreateTableReader<NYT::TNode>(outputTables.Sprav);
    for (; reader->IsValid(); reader->Next()) {
        const NYT::TNode &row = reader->GetRow();
        const TString &name = row[F_NAME].AsString();
        counters[name].Sprav = true;
    }

    reader = clients.ChatNSkills->CreateTableReader<NYT::TNode>(outputTables.Chats);
    for (; reader->IsValid(); reader->Next()) {
        const NYT::TNode &row = reader->GetRow();
        const TString &name = row[F_NAME].AsString();
        counters[name].Chats = true;
    }

    reader = clients.ChatNSkills->CreateTableReader<NYT::TNode>(outputTables.Dialogs);
    for (; reader->IsValid(); reader->Next()) {
        const NYT::TNode &row = reader->GetRow();
        const TString &name = row[F_NAME].AsString();
        counters[name].Dialogs = true;
    }

    reader = clients.ChatNSkills->CreateTableReader<NYT::TNode>(outputTables.Skills);
    for (; reader->IsValid(); reader->Next()) {
        const NYT::TNode &row = reader->GetRow();
        const TString &name = row[F_NAME].AsString();
        counters[name].Skills = true;
    }

    reader = clients.Mascot->CreateTableReader<NYT::TNode>(outputTables.Mascot);
    for (; reader->IsValid(); reader->Next()) {
        const NYT::TNode &row = reader->GetRow();
        const TString &name = row[F_NAME].AsString();
        counters[name].Commercial = true;
    }

    reader = clients.Webmaster->CreateTableReader<NYT::TNode>(outputTables.Webmaster);
    for (; reader->IsValid(); reader->Next()) {
        const NYT::TNode &row = reader->GetRow();
        const TString &name = row[F_NAME].AsString();
        counters[name].Webmaster = true;
    }

    reader = clients.WebmasterVisits->CreateTableReader<NYT::TNode>(outputTables.WebmasterVisits);
    for (; reader->IsValid(); reader->Next()) {
        const NYT::TNode &row = reader->GetRow();
        const TString &name = row[F_NAME].AsString();
        counters[name].WebmasterVisits += 1;
    }

    TDeque<std::pair<size_t, TString>> topOrder;
    TDeque<std::pair<size_t, TString>> commercialTopOrder;
    for (const auto &counter : counters) {
        topOrder.push_back(std::make_pair(counter.second.Clicks, counter.first));
        if (counter.second.Commercial) {
            commercialTopOrder.push_back(std::make_pair(counter.second.Clicks, counter.first));
        }
    }
    std::sort(topOrder.rbegin(), topOrder.rend());
    for (size_t i = 0; i < topOrder.size(); i++) {
        counters[topOrder[i].second].Top = i;
    }
    std::sort(commercialTopOrder.rbegin(), commercialTopOrder.rend());
    for (size_t i = 0; i < commercialTopOrder.size(); i++) {
        counters[commercialTopOrder[i].second].CommercialTop = i;
    }
}

void WriteReport(NYT::IClientBasePtr client, const TMap<TString, TCounters> &counters, const TString &report, const TRangeConfig &rc) {
    const TString rangeName = rc.RangeEnd.ToStroka("%Y-%m-%d");
    auto writer = client->CreateTableWriter<NYT::TNode>(NYT::TRichYPath(report).SortedBy(F_NAME));
    for (const auto &obj : counters) {
        writer->AddRow(NYT::TNode()
            (F_NAME, obj.first)
            (F_CLICKS, obj.second.Clicks)
            //(F_SHARE, obj.second.Share)
            (F_TURBO, obj.second.Turbo)
            (F_TURBO_CLICKS, obj.second.TurboClicks)
            (F_TURBO_PAGES, obj.second.TurboPages)
            (F_CHATS, obj.second.Chats)
            (F_SKILLS, obj.second.Skills)
            (F_DIALOGS, obj.second.Dialogs)
            (F_SPRAV, obj.second.Sprav)
            (F_TOP, obj.second.Top)
            (F_COMMERCIAL, obj.second.Commercial)
            (F_COMMERCIAL_TOP, obj.second.CommercialTop)
            (F_PERIOD, rangeName)
            (F_WEBMASTER, obj.second.Webmaster)
            (F_WEBMASTER_VISITORS, obj.second.WebmasterVisits)
        );
    }
    writer->Finish();
}

void WriteReport(const TClients &clients, TTablePack::Ptr tablePack, const TRangeConfig &rc) {
    if (tablePack->IsClicksTablesReady(clients.Queries) && tablePack->IsTurboTablesReady(clients.Turbo)) {
        LOG_INFO("report %s", rc.Name().data());

        /*
        if (!tx->Exists(tablePack->OutputHosts.Report)) {
            ReadSources(clientQueries, tablePack->OutputHosts, counters);
            WriteReport(clientQueries, counters, tablePack->OutputHosts.Report);
        }
        */

        if (!clients.Sources->Exists(tablePack->OutputOwners.Report)) {
            TMap<TString, TCounters> counters;
            NYT::ITransactionPtr tx = clients.Sources->StartTransaction();
            ReadSources(clients, tablePack->OutputOwners, counters);
            WriteReport(tx, counters, tablePack->OutputOwners.Report, rc);
            tx->Commit();
        }

        LOG_INFO("report %s - done", rc.Name().data());
    }
}
/*
void BuildTurboClicksSource(NYT::IClientBasePtr client, const TMap<TRangeConfig, TTablePack::Ptr> &ranges) {
    THolder<IThreadPool> processQueue(CreateThreadPool(2));
    for (const auto &rcObj : ranges) {
        const TRangeConfig &range = rcObj.first;
        const TTablePack::Ptr &tablePack = rcObj.second;

        if (!tablePack->IsSerpLogsTableSetComplete(range)) {
            if (!range.Current) {
                LOG_WARN("range %s, serp logs is incomplete: %lu / %u", range.Name().data(), tablePack->InputSerpLogs.size(), range.GetDaysCount());
            }
            continue;
        }

        if (tablePack->IsTurboTablesReady(client)) {
            continue;
        }

        processQueue->SafeAddFunc([=, &client]() {
            try {
                const TString &outputHosts = tablePack->OutputHosts.Turbo;
                const TString outputStage1 = outputHosts + "-stage1";
                const TString outputStage2 = outputHosts + "-stage2";

                NYT::ITransactionPtr tx = client->StartTransaction();
                TOpRunner runner(tx);
                for (const TString &table : tablePack->InputSerpLogs) {
                    runner.InputYaMR(table);
                }

                runner
                    .OutputNode(outputStage1)
                    .ReduceBy(F_YAMR_KEY)
                    .SortBy(F_YAMR_KEY, F_YAMR_SUBKEY)
                    .Reduce(new TParseSearchLogsReducer)

                    .InputNode(outputStage1)
                    .OutputNode(outputStage2)
                    .ReduceBy(F_HOST, F_PATH, F_REQID)
                    .MapReduce(nullptr, new TTurboClicksS1Reducer)
                    .Drop(outputStage1)

                    .InputNode(outputStage2)
                    .OutputNode(outputHosts)
                    .ReduceBy(F_NAME)
                    .SortBy(F_NAME, F_PATH)
                    .MapReduce(nullptr, new TTurboClicksS2Reducer)
                    .Drop(outputStage2)

                    .InputNode(outputHosts)
                    .OutputNode(tablePack->OutputOwners.Turbo)
                    .ReduceBy(F_NAME)
                    .MapReduce(new TTurboClicksGetOwnersMapper, new TTurboClicksGetOwnersReducer)
                ;
                tx->Commit();
            } catch(yexception &e) {
                LOG_ERROR("process range %s error: %s", rcObj.first.Name().data(), e.what());
            }
        });
    }
    processQueue->Stop();
}
*/
} //namespace NWebmaster

/**
  * Hitman project https://hitman.yandex-team.ru/projects/WMC-5777
  * Executable hahn://home/webmaster/prod/bin/b2b-top
  */
int main(int argc, const char **argv) {
    setenv("YT_POOL", "robot-webmaster", 1);
    using namespace NWebmaster;
    NYT::Initialize(argc, argv);
    NYTUtils::DisableLogger();

    const TString config_TABLE_TOP_ROOT                             = "//home/webmaster/prod/b2b-top/weekly";
    const TString config_TABLE_TOP_REPORT                           = NYTUtils::JoinPath(config_TABLE_TOP_ROOT, "report");
    const TString config_TABLE_TOP_REPORT_ALL_HOSTS                 = NYTUtils::JoinPath(config_TABLE_TOP_REPORT, "all", "hosts");
    const TString config_TABLE_TOP_REPORT_ALL_OWNERS                = NYTUtils::JoinPath(config_TABLE_TOP_REPORT, "all", "owners");
    //const TString config_TABLE_TOP_REPORT_TURBO_HOSTS             = NYTUtils::JoinPath(config_TABLE_TOP_REPORT, "turbo", "hosts");
    //const TString config_TABLE_TOP_REPORT_TURBO_OWNERS            = NYTUtils::JoinPath(config_TABLE_TOP_REPORT, "turbo", "owners");
    const TString config_TABLE_TOP_SOURCES                          = NYTUtils::JoinPath(config_TABLE_TOP_ROOT, "sources");
    const TString config_TABLE_TOP_SOURCES_CHATS_OWNERS             = NYTUtils::JoinPath(config_TABLE_TOP_SOURCES, "chats", "owners");
    const TString config_TABLE_TOP_SOURCES_CLICKS_HOSTS             = NYTUtils::JoinPath(config_TABLE_TOP_SOURCES, "clicks_v2", "hosts");
    const TString config_TABLE_TOP_SOURCES_CLICKS_OWNERS            = NYTUtils::JoinPath(config_TABLE_TOP_SOURCES, "clicks_v2", "owners");
    const TString config_TABLE_TOP_SOURCES_DIALOGS_OWNERS           = NYTUtils::JoinPath(config_TABLE_TOP_SOURCES, "dialogs", "owners");
    const TString config_TABLE_TOP_SOURCES_MASCOT_OWNERS            = NYTUtils::JoinPath(config_TABLE_TOP_SOURCES, "mascot", "owners");
    const TString config_TABLE_TOP_SOURCES_SKILLS_OWNERS            = NYTUtils::JoinPath(config_TABLE_TOP_SOURCES, "skills", "owners");
    const TString config_TABLE_TOP_SOURCES_SPRAV_OWNERS             = NYTUtils::JoinPath(config_TABLE_TOP_SOURCES, "sprav", "owners");
    const TString config_TABLE_TOP_SOURCES_TURBO_HOSTS              = NYTUtils::JoinPath(config_TABLE_TOP_SOURCES, "turbo_v4", "hosts");
    const TString config_TABLE_TOP_SOURCES_TURBO_OWNERS             = NYTUtils::JoinPath(config_TABLE_TOP_SOURCES, "turbo_v4", "owners");
    const TString config_TABLE_TOP_SOURCES_WEBMASTER_OWNERS         = NYTUtils::JoinPath(config_TABLE_TOP_SOURCES, "webmaster", "owners");
    const TString config_TABLE_TOP_SOURCES_WEBMASTER_VISITS_HOSTS   = NYTUtils::JoinPath(config_TABLE_TOP_SOURCES, "webmaster_visits", "hosts");
    const TString config_TABLE_TOP_SOURCES_WEBMASTER_VISITS_OWNERS  = NYTUtils::JoinPath(config_TABLE_TOP_SOURCES, "webmaster_visits", "owners");

    //const TString config_TABLE_SOURCE_TURBO_HISTORY     = "//home/turborss/production/logs/history";
    const TString config_TABLE_SOURCE_SPRAV             = "//home/webmaster/prod/b2b-top/weekly/raw/sprav";
    const TString config_TABLE_SOURCE_DIALOGS           = "//home/business-chat/prod/dialogs/publish/dialogs";
    const TString config_TABLE_SOURCE_CHATS_AND_SKILLS  = "//home/paskills/skills/stable";
    const TString config_TABLE_SOURCE_MASCOT_CM_ROOT    = "//home/antispam/mascot/history";
    const TString config_TABLE_SOURCE_WEBMASTER         = "//home/webmaster/prod/sitetree";
    //const TString config_TABLE_TOP_ROOT         = "//home/webmaster/prod/export/b2b-top";

    NYT::IClientPtr hahn    = NYT::CreateClient("hahn.yt.yandex.net");
    NYT::IClientPtr arnold  = NYT::CreateClient("arnold.yt.yandex.net");

    TClients clients;
    clients.ChatNSkills     = hahn;
    //clients.Dialogs       = arnold;
    clients.Mascot          = arnold;
    clients.Queries         = arnold;
    clients.Sources         = hahn;
    clients.Sprav           = hahn;
    clients.Turbo           = hahn;
    clients.Webmaster       = arnold;
    clients.WebmasterVisits = hahn;

    NYTUtils::CreatePath(clients.Mascot, config_TABLE_TOP_SOURCES_MASCOT_OWNERS);
    NYTUtils::CreatePath(clients.Queries, config_TABLE_TOP_SOURCES_CLICKS_HOSTS);
    NYTUtils::CreatePath(clients.Queries, config_TABLE_TOP_SOURCES_CLICKS_OWNERS);
    NYTUtils::CreatePath(clients.Turbo, config_TABLE_TOP_SOURCES_TURBO_HOSTS);
    NYTUtils::CreatePath(clients.Turbo, config_TABLE_TOP_SOURCES_TURBO_OWNERS);
    NYTUtils::CreatePath(clients.Queries, config_TABLE_TOP_REPORT_ALL_HOSTS);
    NYTUtils::CreatePath(clients.Queries, config_TABLE_TOP_REPORT_ALL_OWNERS);
    NYTUtils::CreatePath(clients.Sprav, config_TABLE_TOP_SOURCES_SPRAV_OWNERS);
    NYTUtils::CreatePath(clients.ChatNSkills, config_TABLE_TOP_SOURCES_DIALOGS_OWNERS);
    NYTUtils::CreatePath(clients.ChatNSkills, config_TABLE_TOP_SOURCES_SKILLS_OWNERS);
    NYTUtils::CreatePath(clients.ChatNSkills, config_TABLE_TOP_SOURCES_CHATS_OWNERS);
    NYTUtils::CreatePath(clients.Webmaster, config_TABLE_TOP_SOURCES_WEBMASTER_OWNERS);
    NYTUtils::CreatePath(clients.WebmasterVisits, config_TABLE_TOP_SOURCES_WEBMASTER_VISITS_HOSTS);
    NYTUtils::CreatePath(clients.WebmasterVisits, config_TABLE_TOP_SOURCES_WEBMASTER_VISITS_OWNERS);

    TRangeConfig rc = TRangeConfig::Today();
    TMap<TRangeConfig, TTablePack::Ptr> ranges;
    for (int i = 0; i < 20; i++) {
        TTablePack::Ptr pack(new TTablePack);
        pack->OutputHosts.Clicks            = NYTUtils::JoinPath(config_TABLE_TOP_SOURCES_CLICKS_HOSTS, rc.Name());
        pack->OutputHosts.Report            = NYTUtils::JoinPath(config_TABLE_TOP_REPORT_ALL_HOSTS, rc.Name());
        pack->OutputHosts.Turbo             = NYTUtils::JoinPath(config_TABLE_TOP_SOURCES_TURBO_HOSTS, rc.Name());
        pack->OutputHosts.WebmasterVisits   = NYTUtils::JoinPath(config_TABLE_TOP_SOURCES_WEBMASTER_VISITS_HOSTS, rc.Name());
        pack->OutputOwners.Chats            = NYTUtils::JoinPath(config_TABLE_TOP_SOURCES_CHATS_OWNERS, rc.Name());
        pack->OutputOwners.Clicks           = NYTUtils::JoinPath(config_TABLE_TOP_SOURCES_CLICKS_OWNERS, rc.Name());
        pack->OutputOwners.Dialogs          = NYTUtils::JoinPath(config_TABLE_TOP_SOURCES_DIALOGS_OWNERS, rc.Name());
        pack->OutputOwners.Mascot           = NYTUtils::JoinPath(config_TABLE_TOP_SOURCES_MASCOT_OWNERS, rc.Name());
        pack->OutputOwners.Report           = NYTUtils::JoinPath(config_TABLE_TOP_REPORT_ALL_OWNERS, rc.Name());
        pack->OutputOwners.Skills           = NYTUtils::JoinPath(config_TABLE_TOP_SOURCES_SKILLS_OWNERS, rc.Name());
        pack->OutputOwners.Sprav            = NYTUtils::JoinPath(config_TABLE_TOP_SOURCES_SPRAV_OWNERS, rc.Name());
        pack->OutputOwners.Turbo            = NYTUtils::JoinPath(config_TABLE_TOP_SOURCES_TURBO_OWNERS, rc.Name());
        pack->OutputOwners.Webmaster        = NYTUtils::JoinPath(config_TABLE_TOP_SOURCES_WEBMASTER_OWNERS, rc.Name());
        pack->OutputOwners.WebmasterVisits  = NYTUtils::JoinPath(config_TABLE_TOP_SOURCES_WEBMASTER_VISITS_OWNERS, rc.Name());
        ranges[rc] = pack;
        rc = TRangeConfig::Prev(rc);
    }

    LoadClicksTablesList(clients.Queries, ranges);
    BuildClicksSource(clients.Queries, ranges);
    //LoadSerpLogsTablesList(clients.Turbo, ranges);
    //BuildTurboClicksSource(clients.Turbo, ranges);
    //BuildSourceTurbo(clients.Queries, config_TABLE_SOURCE_TURBO_HISTORY, ranges);
    WriteSpravSource(clients.Sprav, config_TABLE_SOURCE_SPRAV, ranges);
    //WriteDialogsSource(clients.Dialogs, clients.Queries, config_TABLE_SOURCE_DIALOGS, ranges);
    WriteDialogsChatsAndSkillsSource(clients.ChatNSkills, config_TABLE_SOURCE_CHATS_AND_SKILLS, ranges);
    BuildMascotCommercialSource(clients.Mascot, config_TABLE_SOURCE_MASCOT_CM_ROOT, ranges);
    BuildWebmasterSource(clients.Webmaster, config_TABLE_SOURCE_WEBMASTER, ranges);
    LoadWebmasterFrontLogsTablesList(clients.WebmasterVisits, ranges);
    BuildWebmasterVisitsSource(clients.WebmasterVisits, ranges);

    for (auto &rcObj : ranges) {
        const TRangeConfig &range = rcObj.first;
        const TTablePack::Ptr &tablePack = rcObj.second;
        WriteReport(clients, tablePack, range);
    }
}
