#include <kernel/hosts/owner/owner.h>

#include <robot/library/yt/static/command.h>
#include <robot/library/yt/static/tags.h>
#include <robot/jupiter/protos/acceptance.pb.h>

#include <util/generic/size_literals.h>

#include <wmconsole/version3/wmcutil/log.h>
#include <wmconsole/version3/wmcutil/regex.h>
#include <wmconsole/version3/wmcutil/thread.h>
#include <wmconsole/version3/wmcutil/url.h>

#include <limits>

#include "config.h"
#include "merge_turbo_sources.h"
#include "field_names.h"

using namespace NJupiter;

namespace NWebmaster {
namespace NTurbo {

static const size_t MAX_ERROR_PER_LISTING = 100;
static const size_t MAX_AUTOPARSER_SAMPLES = 10;

enum ESourceTable {
    TABLE_FEEDS = 0,
    TABLE_BANS,
    TABLE_AUTORELATED,
    TABLE_AUTOMORDA,
    TABLE_APP_REVIEWS_INFO,
    TABLE_AUTOPARSER,
    TABLE_COMMERCE_CATEGORIES,
    TABLE_EXPERIMENTS,
    TABLE_LISTINGS,
    TABLE_MARKET_FEEDS_1, // TODO map <TableIndex, ESourceTable>
    TABLE_MARKET_FEEDS_2,
    //scc
    TURBO_SOURCE_MARKET_CATEGORIES,
    TABLE_TURBO_HOSTS,
    TABLE_TURBO_FEEDS_SETTINGS,
    TABLE_SHOP_CATALOG_STATES,

    TABLE_PREV_DOMAINS_STATE // always last!
};

enum EFeedState {
    FEED_OK,
    FEED_ERROR,
    FEED_WARNING
};

struct TTurboStateSourcesMapper : public NYT::IMapper<NYT::TTableReader<NYT::TNode>, NYT::TTableWriter<NYT::TNode>> {
public:
    void Start(TWriter *) override {
        OwnerCanonizer.LoadTrueOwners();
        OwnerCanonizer.LoadSerpCategOwners();
    }

    void Do(TReader *input, TWriter *output) override {
        static const THashSet<TString> LISTINGS_ERRORS = {
            "bulky.count.categories",
            "bulky.count.vendors",
            "bulky.count.params",
            "bulky.count.param_values",
            "bulky.count.breadcrumbs",
            "bulky.size.params.name",
            "bulky.size.params.unit",
            "bulky.size.params.value",
            "bulky.size.vendors.name",
            "bulky.size.categories.name",
            "bulky.size.listings.url",
            "bulky.size.listings.title"
        };

        for (; input->IsValid(); input->Next()) {
            NYT::TNode row = input->GetRow();
            row[F_SOURCE_TABLE] = input->GetTableIndex();
            switch (input->GetTableIndex()) {
                case TABLE_AUTORELATED:
                case TABLE_AUTOMORDA:
                    // Host -> domain
                    row[FIELD_DOMAIN] = row["Host"];
                    break;
                case TABLE_APP_REVIEWS_INFO:
                    // site -> somain
                    if (NYTUtils::GetNodeFieldOrDefault<TString>(row, "reviews_link", "").empty() &&
                        NYTUtils::GetNodeFieldOrDefault<ui64>(row, "reviews_count", 0) == 0 &&
                        NYTUtils::GetNodeFieldOrDefault<double>(row, "rating", 0.0) == 0.0) {
                        continue;
                    }
                    row[FIELD_DOMAIN] = NUtils::FixDomainPrefix(row["site"].AsString());
                    break;
                case TABLE_AUTOPARSER: {
                    // add all domains up to owner
                    // kondrovo.mirkvartir.ru -> kondrovo.mirkvartir.ru + mirkvartir.ru
                    const TString sourceHost = row["Host"].AsString();
                    row[FIELD_HOST] = sourceHost;
                    TStringBuf host = sourceHost;
                    const TStringBuf owner = OwnerCanonizer.GetHostOwner(host);
                    // for each domain add row
                    size_t idx = host.find(".");
                    while (idx != TString::npos) {
                        host = host.SubStr(idx + 1);
                        row[FIELD_DOMAIN] = host;
                        output->AddRow(row);
                        if (host == owner) {
                            break;
                        }
                        idx = host.find(".");
                    }
                    row[FIELD_DOMAIN] = sourceHost;
                    break;
                }
                case TABLE_COMMERCE_CATEGORIES:
                    // host -> domain
                    row[FIELD_DOMAIN] = row[FIELD_HOST];
                    break;
                case TABLE_EXPERIMENTS:
                    // Owner -> domain
                    row[FIELD_DOMAIN] = row["Owner"];
                    break;
                case TABLE_LISTINGS: {
                    // cut samples and errors
                    NYT::TNode result;
                    result[FIELD_DOMAIN] = NUtils::FixDomainPrefix(NUtils::RemoveScheme(row[FIELD_HOST].AsString()));
                    result["verdict"] = row["verdict"];
                    result[F_SOURCE_TABLE] = row[F_SOURCE_TABLE];
                    NYT::TNode errors = NYT::TNode::CreateList();
                    // flat map samples->errors + errors
                    size_t errorCount = 0;
                    for (const NYT::TNode &sample : row["samples"].AsList()) {
                        for (const NYT::TNode &error : sample["errors"].AsList()) {
                            if (NYTUtils::GetNodeFieldOrDefault<bool>(error, "public", false) &&
                                LISTINGS_ERRORS.contains(error["code"].AsString())) {
                                NYT::TNode errorCopy = error;
                                errorCopy["original_url"] = sample["original_url"];
                                errorCopy["turbo_url"] = sample["turbo_url"];
                                errors.Add(errorCopy);
                                errorCount++;
                            }
                        }
                    }
                    for (const NYT::TNode &error : row["errors"].AsList()) {
                        if (NYTUtils::GetNodeFieldOrDefault<bool>(error, "public", false) &&
                            LISTINGS_ERRORS.contains(error["code"].AsString())) {
                            errors.Add(error);
                            errorCount++;
                            if (errorCount >= MAX_ERROR_PER_LISTING) {
                                break;
                            }
                        }
                    }
                    result["errors"] = errors;
                    row = result;
                    break;
                }
                case TABLE_TURBO_HOSTS:
                    row[FIELD_DOMAIN] = NUtils::FixDomainPrefix(NUtils::RemoveScheme(row["Host"].AsString()));
                    break;
                case TABLE_MARKET_FEEDS_1:
                case TABLE_MARKET_FEEDS_2:
                case TURBO_SOURCE_MARKET_CATEGORIES:
                    row[FIELD_DOMAIN] = NUtils::FixDomainPrefix(NUtils::RemoveScheme(row["host"].AsString()));
                    break;
                case TABLE_SHOP_CATALOG_STATES:
                    row[FIELD_DOMAIN] = row["shop_id"].AsString();
                    break;
            }
            if (row[FIELD_DOMAIN].HasValue() && !row[FIELD_DOMAIN].AsString().empty()) {
                output->AddRow(row);
            }
        }
    }
private:
    TOwnerCanonizer OwnerCanonizer;
};

REGISTER_MAPPER(TTurboStateSourcesMapper)

struct TTurboStateSourcesReducer : public NYT::IReducer<NYT::TTableReader<NYT::TNode>, NYT::TTableWriter<NYT::TNode>> {
public:
    void Do(TReader *input, TWriter *output) override {
        static const int OUTPUT_STATE = 0;
        static const int OUTPUT_CHANGES = 1;
        static const THashMap<TString, THashMap<EFeedState, TString>> NOTIFICATIONS_BY_BAD_FEEDS{
            {SOURCE_RSS, {
                             {FEED_ERROR, "TURBO_RSS_ERROR"},
                             {FEED_WARNING, "TURBO_RSS_WARNING"}
                         }},
            {SOURCE_YML, {
                             {FEED_ERROR, "TURBO_YML_ERROR"},
                             {FEED_WARNING, "TURBO_YML_WARNING"}
                         }},
        };
        static const TVector<TString> BAN_NOTIFICATIONS {"TURBO_FEED_BAN", "TURBO_DOCUMENT_BAN"};
        static const THashMap<TString, TString> NOTIFICATIONS_BY_BAN_TYPE {
            {"feed_url", "TURBO_FEED_BAN"},
            {"feed_domain", "TURBO_FEED_BAN"},
            {"doc_url", "TURBO_DOCUMENT_BAN"},
            {"doc_domain", "TURBO_DOCUMENT_BAN"}
        };
        static const THashSet<TString> VERDICTS_FOR_TURNING_ON {"OK", "WARN"};

        ui64 now = Now().MilliSeconds();

        NYT::TNode state;
        state[FIELD_DOMAIN] = input->GetRow()[FIELD_DOMAIN];
        state[F_RSS_FEEDS] = NYT::TNode::CreateList();
        state[F_YML_FEEDS] = NYT::TNode::CreateList();
        state[F_BANS] = NYT::TNode::CreateList();
        state[F_MARKET_FEEDS] = NYT::TNode::CreateList();
        state[F_PROBLEMS] = NYT::TNode::CreateMap();
        state[F_PREMODERATION_RESULT] = NYT::TNode::CreateMap();
        state[F_BANNED_SCC] = NYT::TNode::CreateMap();

        THashMap<size_t, TVector<NYT::TNode>> apSamplesBySubdomainLevel;
        size_t domainLevel = NUtils::GetDomainLevel(state[FIELD_DOMAIN].AsString());

        bool hasActiveYmlFeeds = false;
        THashMap<TString, THashMap<EFeedState, TVector<NYT::TNode>>> badFeeds;
        THashMap<TString, TVector<NYT::TNode>> bansByType;
        NYT::TNode prevState;
        NYT::TNode turboHosts;
        NYT::TNode marketCategories;
        TVector<NYT::TNode> feedsYmlSettings;
        for (; input->IsValid(); input->Next()) {
            // collect all info depending on source
            NYT::TNode row = input->GetRow();
            size_t sourceTable = row[F_SOURCE_TABLE].AsUint64();
            switch (sourceTable) {
                case TABLE_FEEDS:
                    if (row[FIELD_TYPE].AsString() == SOURCE_RSS) {
                        state[F_RSS_FEEDS].Add(row);
                    } else {
                        state[F_YML_FEEDS].Add(row);
                        hasActiveYmlFeeds |= row["active"].AsBool();
                    }
                    badFeeds[row[FIELD_TYPE].AsString()][GetFeedState(row)].push_back(row);
                    break;
                case TABLE_BANS:
                    state[F_BANS].Add(row);
                    bansByType[NOTIFICATIONS_BY_BAN_TYPE.at(row["ban_type"].AsString())].push_back(row);
                    break;
                case TABLE_AUTORELATED:
                    state[F_AUTORELATED_SAMPLES] = row["Samples"];
                    break;
                case TABLE_AUTOMORDA:
                    state[F_AUTOMORDA_SAMPLES] = row["Samples"];
                    state[F_AUTOMORDA_STATUS] = row["Status"];
                    break;
                case TABLE_APP_REVIEWS_INFO:
                    state[F_APP_REVIEWS_INFO] = NYT::TNode()
                        ("rating", NYTUtils::GetNodeFieldOrDefault<double>(row, "rating", 0))
                        ("link", row["reviews_link"])
                        ("count", row["reviews_count"])
                        ("rate_count", row["count"]);
                    break;
                case TABLE_AUTOPARSER: {
                    size_t subdomainLevel = NUtils::GetDomainLevel(row[FIELD_HOST].AsString());
                    apSamplesBySubdomainLevel[domainLevel - subdomainLevel].push_back(row["Samples"]);
                    break;
                }
                case TABLE_COMMERCE_CATEGORIES:
                    state[F_COMMERCE_CATEGORIES] = ProcessCommerceCategories(row["categories"]);
                    break;
                case TABLE_EXPERIMENTS:
                    state[F_EXPERIMENT] = row["Experiment"];
                    break;
                case TABLE_LISTINGS:
                    state[F_LISTINGS_INFO] = row;
                    break;
                case TABLE_MARKET_FEEDS_1:
                case TABLE_MARKET_FEEDS_2:
                    state[F_MARKET_FEEDS].Add(row);
                    break;
                case TURBO_SOURCE_MARKET_CATEGORIES:
                    marketCategories = row;
                    break;
                case TABLE_SHOP_CATALOG_STATES:
                    state[F_SHOP_STATES] = row;
                    break;
                case TABLE_PREV_DOMAINS_STATE:
                    prevState = row;
                    break;
                case TABLE_TURBO_HOSTS:
                    turboHosts = row;
                    break;
                case TABLE_TURBO_FEEDS_SETTINGS:
                    if (row["type"] == "YML") {
                        feedsYmlSettings.push_back(row);
                    }
                    break;
            }
        }

        if (!turboHosts.HasValue()) {
            return;
        }

        THashMap<TString, THashMap<EFeedState, TVector<NYT::TNode>>> prevBadFeeds;
        if (prevState.HasValue()) {
            for (const NYT::TNode &feed : prevState[F_RSS_FEEDS].AsList()) {
                prevBadFeeds[feed[FIELD_TYPE].AsString()][GetFeedState(feed)].push_back(feed);
            }
            for (const NYT::TNode &feed : prevState[F_YML_FEEDS].AsList()) {
                prevBadFeeds[feed[FIELD_TYPE].AsString()][GetFeedState(feed)].push_back(feed);
            }
        }
        // generate notifications (feed errors and warnings)
        for (const auto &notificationsByType : NOTIFICATIONS_BY_BAD_FEEDS) {
            const TString &feedType = notificationsByType.first;
            for (const auto &notificationBySeverity : notificationsByType.second) {
                const EFeedState &feedState = notificationBySeverity.first;
                const TString &notification = notificationBySeverity.second;
                if (badFeeds[feedType][feedState].empty()) {
                    continue;
                }
                // store problem
                NYT::TNode problem;
                problem[F_LAST_UPDATE] = now;
                problem[F_ACTUAL_SINCE] = GetProblemActualSince(prevState, notification);
                problem[FIELD_DATA] = NYT::TNode::CreateList(badFeeds[feedType][feedState]);
                state[F_PROBLEMS][notification] = problem;
                // store changes if needed
                if (prevBadFeeds[feedType][feedState].empty()) {
                    output->AddRow(NYT::TNode()
                        (FIELD_DOMAIN, state[FIELD_DOMAIN])
                        (FIELD_TYPE, notification)
                        (F_LAST_UPDATE, problem[F_LAST_UPDATE])
                        (F_ACTUAL_SINCE, problem[F_ACTUAL_SINCE])
                        (FIELD_DATA, problem[FIELD_DATA]),
                        OUTPUT_CHANGES
                    );
                }
            }
        }
        // generate notifications about bans
        THashMap<TString, TVector<NYT::TNode>> prevBansByType;
        if (prevState.HasValue()) {
            for (const NYT::TNode &ban : prevState[F_BANS].AsList()) {
                prevBansByType[NOTIFICATIONS_BY_BAN_TYPE.at(ban["ban_type"].AsString())].push_back(ban);
            }
        }
        for (const TString& banNotification : BAN_NOTIFICATIONS) {
            if (bansByType[banNotification].empty()) {
                continue;
            }
            // store problem
            NYT::TNode problem;
            problem[F_LAST_UPDATE] = now;
            problem[F_ACTUAL_SINCE] = GetProblemActualSince(prevState, banNotification);
            problem[FIELD_DATA] = NYT::TNode::CreateList(bansByType[banNotification]);
            state[F_PROBLEMS][banNotification] = problem;
            // store changes
            if (prevBansByType[banNotification].empty()) {
                output->AddRow(NYT::TNode()
                    (FIELD_DOMAIN, state[FIELD_DOMAIN])
                    (FIELD_TYPE, banNotification)
                    (F_LAST_UPDATE, problem[F_LAST_UPDATE])
                    (F_ACTUAL_SINCE, problem[F_ACTUAL_SINCE])
                    (FIELD_DATA, problem[FIELD_DATA]),
                    OUTPUT_CHANGES
                );
            }
        }
        // generate notification about new autoparser
        if (state.HasKey(F_AUTOPARSER_SAMPLES) && (!prevState.HasValue() || !prevState[F_AUTOPARSER_SAMPLES].HasValue())) {
            output->AddRow(NYT::TNode()
                    (FIELD_DOMAIN, state[FIELD_DOMAIN])
                    (FIELD_TYPE, "TURBO_AUTOPARSED_PAGES_APPEARED")
                    (FIELD_DATA, state[F_AUTOPARSER_SAMPLES]),
                    OUTPUT_CHANGES
            );
        }
        // save samples from domain and subdomains (aka pushUpAutoparsedSamples)
        state[F_AUTOPARSER_SAMPLES] =  CollectSamplesFromDomainAndSubdomains(apSamplesBySubdomainLevel);

        // generate notification about listing errors and available listings
        bool listingsAvailable = false, prevListingsAvailable = false;
        bool hasListingErrors = false, prevHasListingErrors = false;
        if (state.HasKey(F_LISTINGS_INFO)) {
            listingsAvailable = VERDICTS_FOR_TURNING_ON.contains(state[F_LISTINGS_INFO]["verdict"].AsString());
            hasListingErrors = state[F_LISTINGS_INFO]["errors"].Size() > 0 && hasActiveYmlFeeds;
        }
        if (prevState.HasValue() && prevState[F_LISTINGS_INFO].HasValue()) {
            prevListingsAvailable = VERDICTS_FOR_TURNING_ON.contains(prevState[F_LISTINGS_INFO]["verdict"].AsString());
            prevHasListingErrors = prevState[F_LISTINGS_INFO]["errors"].Size() > 0 && hasActiveYmlFeeds;
        }
        if (listingsAvailable && !prevListingsAvailable) {
            output->AddRow(NYT::TNode()
                    (FIELD_DOMAIN, state[FIELD_DOMAIN])
                    (FIELD_TYPE, "TURBO_LISTINGS_AVAILABLE")
                    (FIELD_DATA, state[F_LISTINGS_INFO]),
                    OUTPUT_CHANGES
            );
        }
        if (hasListingErrors) {
            const TString notificationType = "TURBO_LISTING_ERROR";
            // store problem
            NYT::TNode problem;
            problem[F_LAST_UPDATE] = now;
            problem[F_ACTUAL_SINCE] = GetProblemActualSince(prevState, notificationType);
            problem[FIELD_DATA] = state[F_LISTINGS_INFO];
            state[F_PROBLEMS][notificationType] = problem;
            if (hasListingErrors && !prevHasListingErrors) {
                output->AddRow(NYT::TNode()
                        (FIELD_DOMAIN, state[FIELD_DOMAIN])
                        (FIELD_TYPE, notificationType)
                        (F_LAST_UPDATE, problem[F_LAST_UPDATE])
                        (F_ACTUAL_SINCE, problem[F_ACTUAL_SINCE])
                        (FIELD_DATA, state[F_LISTINGS_INFO]),
                        OUTPUT_CHANGES
                );
            }
        }
        TString prevSccStatus = "UNKNOWN";
        TString curSccStatus = "UNKNOWN";

        if (prevState.HasValue() && prevState[F_PREMODERATION_RESULT].HasValue() && prevState[F_PREMODERATION_RESULT]["wmc_status"].HasValue()) {
            prevSccStatus =  prevState[F_PREMODERATION_RESULT]["wmc_status"].AsString();
        }

        TVector<NYT::TNode> reasons;
        bool hasBannedFeed = false;
        bool hasFailedFeed = false;
        bool hasInProgress = false;
        bool allFeedsPassed = !feedsYmlSettings.empty();
        int countEnabledFeeds = 0;
        for (const auto &item : feedsYmlSettings) {
            auto status = item["status_scc"];
            if (!status.IsNull() && status.AsString() == "BANNED") {
                hasBannedFeed = true;
                reasons.push_back(NYT::TNode()("comment", item["error_scc"]));
            }
            if (!status.IsNull() && (status.AsString() == "FAILED" || status.AsString() == "DISABLED_BY_PINGER")) {
                reasons.push_back(NYT::TNode()("comment", item["error_scc"]));
            }
            if (!item["active"].AsBool()) {
                continue;
            }
            countEnabledFeeds++;
            if (!status.IsNull() && (status.AsString() == "FAILED" || status.AsString() == "DISABLED_BY_PINGER")) {
                hasFailedFeed = true;
            }
            if (!status.IsNull() && (status.AsString() == "IN_PROGRESS")) {
                hasInProgress = true;
            }
            if (status.IsNull() || status.AsString() != "SUCCESS") {
                allFeedsPassed = false;
            }
        }
        if (countEnabledFeeds == 0) {
            allFeedsPassed = false;
        }

        bool outerCart = turboHosts["ProductInfo"].HasValue() && turboHosts["ProductInfo"]["cart_url"].HasValue();
        if (outerCart) {
            curSccStatus = "OUTER_CART";
        } else if (hasBannedFeed) {
            curSccStatus = "BANNED";
        } else if (allFeedsPassed) {
            curSccStatus = "PASS";
        } else if (hasFailedFeed) {
            curSccStatus = "FAILED";
        } else {
            curSccStatus = prevSccStatus;
        }

        state[F_PREMODERATION_RESULT]["wmc_status"] = curSccStatus;
        state[F_PREMODERATION_RESULT]["status"] = curSccStatus;

        if (reasons.empty() && prevState.HasValue() && prevState[F_PREMODERATION_RESULT].HasValue()) {
            state[F_PREMODERATION_RESULT]["items"] = prevState[F_PREMODERATION_RESULT]["items"];
        } else {
            auto problems = NYT::TNode()("problems", NYT::TNode::CreateList(reasons));
            state[F_PREMODERATION_RESULT]["items"] = NYT::TNode::CreateList({problems});
        }


        TString sccNotificationType = "";
        if (prevSccStatus == "BANNED" && curSccStatus != "BANNED") {
            sccNotificationType = "TURBO_SCC_UNBANNED";
        }
        if (prevSccStatus != "BANNED" && curSccStatus == "BANNED") {
            sccNotificationType = "TURBO_SCC_BANNED";
        }
        //
        if ((prevSccStatus == "UNKNOWN" || prevSccStatus == "PASS" || !hasFailedFeed) && curSccStatus == "FAILED") {
            sccNotificationType = "TURBO_SCC_FAILED";
        }
        if ((prevSccStatus == "UNKNOWN" || prevSccStatus == "IN_PROGRESS") && curSccStatus == "PASS") {
            sccNotificationType = "TURBO_SCC_PASS";
        }

        if (!sccNotificationType.empty()) {
            output->AddRow(NYT::TNode()
                                   (FIELD_DOMAIN, state[FIELD_DOMAIN])
                                   (FIELD_TYPE, sccNotificationType)
                                   (FIELD_DATA, NYT::TNode()("current_status", curSccStatus)
                                           ("previously_status", prevSccStatus)
                                   ),
                                   OUTPUT_CHANGES
            );
        }

        output->AddRow(state, OUTPUT_STATE);
    }

    ui64 GetProblemActualSince(NYT::TNode prevState, const TString &problemType) {
        if (prevState.HasValue() && prevState.HasKey(F_PROBLEMS) && prevState[F_PROBLEMS].HasKey(problemType)) {
            return prevState[F_PROBLEMS][problemType][F_ACTUAL_SINCE].AsUint64();
        } else {
            return Now().MilliSeconds();;
        }
    }

    bool HasAcceptableCountOfImageErrors(const NYT::TNode &feed) {
        static const THashSet<TString> IMAGES_ERRORS = {"Images.FetchError", "Images.Unavailable", "Skip.Images"};
        static const double MAX_IGNORED_IMAGES_ERRORS_SHARE = 0.1;
        static const size_t MAX_IGNORED_IMAGES_ERRORS_COUNT = 5;

        const NYT::TNode &data = feed[FIELD_DATA];
        if (feed["active"].AsBool() == false || !data[FIELD_STATS].HasValue()) {
            return data[FIELD_ERRORS].IsUndefined() || data[FIELD_ERRORS].Size() == 0;
        }
        size_t imageErrors = 0;
        THashSet<TString> itemsWithImageErrors;
        for (const NYT::TNode &error : data[FIELD_ERRORS].AsList()) {
            const TString &code = error[FIELD_CODE].AsString();
            if (IMAGES_ERRORS.contains(code)) {
                imageErrors++;
                if (error[FIELD_DETAILS].HasValue() && error[FIELD_DETAILS][FIELD_ITEM_URL].HasValue()) {
                    itemsWithImageErrors.insert(error[FIELD_DETAILS][FIELD_ITEM_URL].AsString());
                }
            } else {
                imageErrors = std::numeric_limits<size_t>::max();
                break;
            }
        }
        int total = 0;
        total += NYTUtils::GetNodeFieldOrDefault<i64>(data[FIELD_STATS], "error", 0);
        total += NYTUtils::GetNodeFieldOrDefault<i64>(data[FIELD_STATS], "info", 0);
        total += NYTUtils::GetNodeFieldOrDefault<i64>(data[FIELD_STATS], "ok", 0);
        total += NYTUtils::GetNodeFieldOrDefault<i64>(data[FIELD_STATS], "warning", 0);
        double imageErrorsShare = itemsWithImageErrors.size() / (double) total;
        return imageErrors <= MAX_IGNORED_IMAGES_ERRORS_COUNT && imageErrorsShare <= MAX_IGNORED_IMAGES_ERRORS_SHARE;
    }

    EFeedState GetFeedState(const NYT::TNode &feed) {
        const NYT::TNode &data = feed[FIELD_DATA];
        // нет ни одного турбо-урла - ошибка
        if ((!data[FIELD_ITEMS].HasValue() || data[FIELD_ITEMS].Size() == 0) &&
            (!data[FIELD_STATS].HasValue() || data[FIELD_STATS]["ok"].AsInt64() == 0)) {
            return FEED_ERROR;
        }
        // если есть ошибки - предупреждение
        if (!HasAcceptableCountOfImageErrors(feed)) {
            return FEED_WARNING;
        }
        // иначе все ок
        return FEED_OK;
    }

    NYT::TNode CollectSamplesFromDomainAndSubdomains(const THashMap<size_t, TVector<NYT::TNode>> &samplesBySubdomain) {
        NYT::TNode result = NYT::TNode::CreateList();
        size_t count = 0; //
        for (const auto &levelAndSamples : samplesBySubdomain) {
            size_t index = 0;
            bool addedAnySample = true;
            while (addedAnySample) {
                addedAnySample = false;
                for (const NYT::TNode &samples : levelAndSamples.second) {
                    if (index < samples.Size()) {
                        result.Add(samples.At(index));
                        count++;
                        addedAnySample = true;
                    }
                    if (count >= MAX_AUTOPARSER_SAMPLES) {
                        return result;
                    }
                }
                index++;
            }
        }
        return result;
    }

    /**
     * Sort categories and convert list to tree
     */
    NYT::TNode ProcessCommerceCategories(NYT::TNode &rawCategories) {
        TVector<NYT::TNode> &rawCategoriesList = rawCategories.AsList();
        // sorting
        std::sort(rawCategoriesList.begin(), rawCategoriesList.end(), [&](const NYT::TNode &a, const NYT::TNode &b) {
            return a["order"].AsInt64() > b["order"].AsInt64();
        });
        // by id
        THashMap<TString, const NYT::TNode*> itemsById;
        THashMap<TString, TVector<TString>> childrenById;
        for (const NYT::TNode &item : rawCategoriesList) {
            const TString id = item["id"].AsString();
            const TString parentId = NYTUtils::GetNodeFieldOrDefault<TString>(item, "parent_id", "");
            itemsById[id] = &item;
            childrenById[parentId].push_back(id);
        }
        return ProcessItems("", itemsById, childrenById);
    }

    NYT::TNode ProcessItems(const TString &parentId, const THashMap<TString, const NYT::TNode*> &itemsById,
            const THashMap<TString, TVector<TString>> &childrenById) {
        NYT::TNode result = NYT::TNode::CreateList();
        if (!childrenById.contains(parentId)) {
            return result;
        }
        for (const TString &childId : childrenById.at(parentId)) {
            const NYT::TNode *item = itemsById.at(childId);
            NYT::TNode processedNode;
            processedNode["id"] = childId;
            processedNode["label"] = item->At("label");
            processedNode["turboUrl"] = item->At("turbo_url");
            processedNode["submenu"] = ProcessItems(childId, itemsById, childrenById);
            if (item->HasKey("original_url")) {
                processedNode["url"] = item->At("original_url");
            }
            result.Add(processedNode);
        }
        return result;
    }

};
REGISTER_REDUCER(TTurboStateSourcesReducer)

int MergeTurboSources(int, const char **) {
    const auto& config = TConfig::CInstance();
    NYT::IClientPtr client = NYT::CreateClient(config.MR_TURBO_SERVER_HOST);
    NYT::ITransactionPtr tx = client->StartTransaction();

    LOG_INFO("About to merge turbo sources");
    TString latestListingsTable;
    if (!NYTUtils::GetLatestTable(tx, config.TURBO_SOURCE_LISTINGS, latestListingsTable)) {
        LOG_ERROR("Could not load latest listings table");
        ythrow yexception() << "Could not load latest listings table";
    }

    NYT::TRichYPath domainsState(config.TABLE_DOMAINS_STATE);
    domainsState.Schema(NYT::TTableSchema()
                          .AddColumn(FIELD_DOMAIN, NYT::EValueType::VT_STRING)
                          .AddColumn(F_RSS_FEEDS, NYT::EValueType::VT_ANY)
                          .AddColumn(F_YML_FEEDS, NYT::EValueType::VT_ANY)
                          .AddColumn(F_BANS, NYT::EValueType::VT_ANY)
                          .AddColumn(F_AUTORELATED_SAMPLES, NYT::EValueType::VT_ANY)
                          .AddColumn(F_AUTOMORDA_SAMPLES, NYT::EValueType::VT_ANY)
                          .AddColumn(F_AUTOMORDA_STATUS, NYT::EValueType::VT_STRING)
                          .AddColumn(F_APP_REVIEWS_INFO, NYT::EValueType::VT_ANY)
                          .AddColumn(F_AUTOPARSER_SAMPLES, NYT::EValueType::VT_ANY)
                          .AddColumn(F_COMMERCE_CATEGORIES, NYT::EValueType::VT_ANY)
                          .AddColumn(F_EXPERIMENT, NYT::EValueType::VT_STRING)
                          .AddColumn(F_LISTINGS_INFO, NYT::EValueType::VT_ANY)
                          .AddColumn(F_MARKET_FEEDS, NYT::EValueType::VT_ANY)
                          .AddColumn(F_PREMODERATION_RESULT, NYT::EValueType::VT_ANY)
                          .AddColumn(F_BANNED_SCC, NYT::EValueType::VT_ANY)
                          .AddColumn(F_PROBLEMS, NYT::EValueType::VT_ANY)
                          .AddColumn(F_SHOP_STATES, NYT::EValueType::VT_ANY));

    const TString changesTableName = NYTUtils::JoinPath(config.TABLE_DOMAINS_STATE_CHANGES_ROOT, ToString(Now().MilliSeconds()));
    NYT::TRichYPath changesTable(changesTableName);
    changesTable.Schema(NYT::TTableSchema()
                          .AddColumn(FIELD_DOMAIN, NYT::EValueType::VT_STRING)
                          .AddColumn(FIELD_TYPE, NYT::EValueType::VT_STRING)
                          .AddColumn(F_LAST_UPDATE, NYT::EValueType::VT_UINT64)
                          .AddColumn(F_ACTUAL_SINCE, NYT::EValueType::VT_UINT64)
                          .AddColumn(FIELD_DATA, NYT::EValueType::VT_ANY));

    TMapReduceCmd<TTurboStateSourcesMapper, TTurboStateSourcesReducer> mapReduceCmd(tx);
    // order is IMPORTANT
    mapReduceCmd
        .Input<NYT::TNode>(config.TABLE_FEEDS_BY_DOMAIN)
        .Input<NYT::TNode>(config.TABLE_BANS_STATE)
        .Input<NYT::TNode>(config.TABLE_SOURCE_AUTORELATED)
        .Input<NYT::TNode>(config.TABLE_SOURCE_AUTOMORDA)
        .Input<NYT::TNode>(config.TABLE_SOURCE_APP_REVIEWS_INFO)
        .Input<NYT::TNode>(config.TURBO_SOURCE_AUTOPARSER)
        .Input<NYT::TNode>(config.TURBO_SOURCE_COMMERCE_CATEGORIES)
        .Input<NYT::TNode>(config.TURBO_SOURCE_EXPERIMENTS)
        .Input<NYT::TNode>(latestListingsTable)
        .Input<NYT::TNode>(config.TURBO_SOURCE_MARKET_FEEDS_1)
        .Input<NYT::TNode>(config.TURBO_SOURCE_MARKET_FEEDS_2)
        .Input<NYT::TNode>(config.TURBO_SOURCE_MARKET_CATEGORIES)
        .Input<NYT::TNode>(config.TABLE_TURBO_CONFIG)
        .Input<NYT::TNode>(config.TABLE_SOURCE_TURBO_FEEDS_SETTINGS)
        .Input<NYT::TNode>(config.TURBO_SOURCE_SHOP_CATALOG_STATES);

    if (tx->Exists(domainsState.Path_)) {
        mapReduceCmd.Input<NYT::TNode>(domainsState);
    }
    mapReduceCmd
        .Output<NYT::TNode>(domainsState)
        .Output<NYT::TNode>(changesTable)
        .MapperMemoryLimit(2_GBs)
        .ReduceBy(FIELD_DOMAIN)
        .Do();

    TSortCmd<NYT::TNode>(tx)
        .Input<NYT::TNode>(domainsState.Path_)
        .Output<NYT::TNode>(domainsState.Path_)
        .By(FIELD_DOMAIN)
        .Do();

    TSortCmd<NYT::TNode>(tx)
        .Input<NYT::TNode>(changesTable.Path_)
        .Output<NYT::TNode>(changesTable.Path_)
        .By(FIELD_DOMAIN)
        .Do();

    tx->Commit();

    return 0;
}

} //namespace NTurbo
} //namespace NWebmaster

