#include <limits>

#include <util/generic/size_literals.h>
#include <util/charset/wide.h>
#include <util/string/escape.h>

#include <robot/jupiter/protos/acceptance.pb.h>
#include <robot/library/yt/static/table.h>

#include <wmconsole/version3/library/jupiter/jupiter.h>
#include <wmconsole/version3/wmcutil/hostid.h>
#include <wmconsole/version3/wmcutil/log.h>
#include <wmconsole/version3/wmcutil/string.h>
#include <wmconsole/version3/wmcutil/url.h>
#include <wmconsole/version3/wmcutil/yt/yt_runner.h>
#include <wmconsole/version3/wmcutil/yt/yt_utils.h>

#include <wmconsole/version3/processors/indexing/conf/yt.h>

#include "fields.h"
#include "monitor.h"
#include "schemes.h"
#include "task_update.h"

namespace NWebmaster {
namespace NImportantUrls {

struct TImportantUrlsPathConvertMapper : public NYT::IMapper<NYT::TTableReader<NYT::TNode>, NYT::TTableWriter<NYT::TNode>> {
    struct TImportantUrlsRecord {
        Y_SAVELOAD_DEFINE(Host, Path)

        TImportantUrlsRecord() = default;

        TImportantUrlsRecord(const TString &url) {
            SplitUrlToHostAndPath(url, Host, Path);
        }

        TImportantUrlsRecord(const TString &host, const TString &path)
            : Host(host)
            , Path(path)
        {
        }

        bool operator<(const TImportantUrlsRecord& rhs) const {
            if (Host == rhs.Host) {
                return Path < rhs.Path;
            }
            return Host < rhs.Host;
        }

    public:
        TString Host;
        TString Path;
    };

    void Do(TReader *input, TWriter *output) override {
        static NYT::TNode nullNode = NYT::TNode::CreateEntity();
        for (; input->IsValid(); input->Next()) {
            const NYT::TNode &row = input->GetRow();
            const TString url = row[F_URL].AsString();
            const NYT::TNode userId = row.HasKey(F_USER_ID) ? row[F_USER_ID] : nullNode;
            TImportantUrlsRecord record(url);
            output->AddRow(NYT::TNode()
                (F_HOST, record.Host)
                (F_PATH, record.Path)
                (F_URL, url)
                (F_TABLE_TIMESTAMP, 0) // for history compressing
                (F_USER_ID, userId)
            );
        }
    }
};

REGISTER_MAPPER(TImportantUrlsPathConvertMapper)

struct TImportantUrlsReducer : public NYT::IReducer<NYT::TTableReader<NYT::TNode>, NYT::TTableWriter<NYT::TNode>> {
    Y_SAVELOAD_JOB(InputConfig, MarkDescriptionChanged)

public:
    TImportantUrlsReducer() = default;
    TImportantUrlsReducer(const TDeque<TImportantUrlsConfig> &inputConfig,
                          const bool markDescriptionChanged = false) // WMC-5446
        : InputConfig(inputConfig)
        , MarkDescriptionChanged(markDescriptionChanged)
    {
    }

    //reduce by Host, Path
    void Do(TReader *input, TWriter *output) override {
        const int OUTPUT_TABLENO_SNAPSHOT   = 0;
        const int OUTPUT_TABLENO_UPDATES    = 1;

        const char *F_IN_TITLE = "TitleRawUTF8";
        const char *F_IN_DESCRIPTION = "MetaDescription";
        static const NYT::TNode nullNode = NYT::TNode::CreateEntity();

        struct TJupiterRecord {
            TJupiterRecord() = default;
            TJupiterRecord(time_t timestamp, const NYT::TNode &row)
                : Timestamp(timestamp)
                , Row(row)
            {
            }

        public:
            size_t Timestamp = 0;
            NYT::TNode Row;
        };

        TMap<size_t, TJupiterRecord> jupiterRows;
        TMap<time_t, NYT::TNode> spreadRows;
        bool gotContentAttrs = false;
        NYT::TNode title, description;
        TString host, path, url;
        NYT::TNode prevSnapshot;
        NYT::TNode fastBanNode = nullNode;
        bool importantUrl = false;
        bool isTurboPageUrl = false;

        for (; input->IsValid(); input->Next()) {
            const NYT::TNode &row = input->GetRow();
            const TImportantUrlsConfig &config = InputConfig[input->GetTableIndex()];

            if (!importantUrl && config.Type != TImportantUrlsConfig::E_TABLE_IMPORTANT_URLS) {
                continue;
            }

            switch(config.Type) {
            case TImportantUrlsConfig::E_TABLE_IMPORTANT_URLS:
                host = row[F_HOST].AsString();
                path = row[F_PATH].AsString();
                url = row[F_URL].AsString();
                importantUrl = true;
                break;
            case TImportantUrlsConfig::E_TABLE_SNAPSHOT:
                prevSnapshot = row;
                break;
            case TImportantUrlsConfig::E_TABLE_FAST_BAN:
                fastBanNode = row;
                break;
            case TImportantUrlsConfig::E_TABLE_JUPITER_BASE: {
                    time_t lastAccess = NYTUtils::FromNodeOrDefault<ui64>(row[F_LAST_ACCESS], 0);
                    jupiterRows[lastAccess] = TJupiterRecord(config.Timestamp, row);
                }
                break;
            case TImportantUrlsConfig::E_TABLE_SPREAD_EXPORT: {
                    time_t lastAccess = NYTUtils::FromNodeOrDefault<ui64>(row[F_LAST_ACCESS], 0);
                    spreadRows[lastAccess] = row;
                }
                break;
            case TImportantUrlsConfig::E_TABLE_CALLISTO_BASE: {
                    time_t lastAccess = NYTUtils::FromNodeOrDefault<ui64>(row[F_LAST_ACCESS], 0);
                    if (!jupiterRows.contains(lastAccess)){
                        jupiterRows[lastAccess] = TJupiterRecord(row["insert_date"].AsUint64(), row);
                        gotContentAttrs = true;
                        title = row[F_IN_TITLE];
                        description = row[F_IN_DESCRIPTION];
                        // WMCSUPPORT-2440 truncate long title/descriptions
                        if (!title.IsNull()) {
                            title = NUtils::Abbreviate(title.AsString(), 1024);
                        }
                        if (!description.IsNull()) {
                            description = NUtils::Abbreviate(description.AsString(), 1024);
                        }
                    }
            }
            break;
            case TImportantUrlsConfig::E_TABLE_CONTENT_ATTRS:
                gotContentAttrs = true;
                title = row[F_IN_TITLE];
                description = row[F_IN_DESCRIPTION];
                // WMCSUPPORT-2440 truncate long title/descriptions
                if (!title.IsNull()) {
                    title = NUtils::Abbreviate(title.AsString(), 1024);
                }
                if (!description.IsNull()) {
                    description = NUtils::Abbreviate(description.AsString(), 1024);
                }
                break;
            case TImportantUrlsConfig::E_TABLE_TURBO_PAGE:
                isTurboPageUrl = true;
                break;
            default:
                ythrow yexception() << "unknown table passed";
            }
        }

        if (!importantUrl) {
            return;
        }

        NYT::TNode currSnapshot;
        currSnapshot[F_HOST] = host;
        currSnapshot[F_PATH] = path;
        currSnapshot[F_URL] = url;

        currSnapshot[F_FOUND_BASE] = !jupiterRows.empty();
        currSnapshot[F_FOUND_SPREAD] = !spreadRows.empty();
        currSnapshot[F_IS_TURBO_PAGE] = isTurboPageUrl;

        if (gotContentAttrs) {
            currSnapshot[F_TITLE] = title;
            currSnapshot[F_DESCRIPTION] = description;
        }

        if (!jupiterRows.empty()) {
            const TJupiterRecord &record = jupiterRows.rbegin()->second;
            const NYT::TNode &row = record.Row;
            const NYT::TNode &urlStatus = row[F_URL_STATUS];
            const bool isBadMimeImage = !NYTUtils::IsNodeNull(urlStatus)
                                  && urlStatus.AsUint64() == NJupiter::EAcceptanceUrlForWebMasterSimpleStatus::AUFWSS_BAD_MIME_TYPE
                                  && IsImagePathExtension(path);
            if (isBadMimeImage) {
                currSnapshot[F_FOUND_BASE] = false;
            } else {
                currSnapshot[F_JUPITER_TIMESTAMP] = record.Timestamp;
                for (const auto &columnName : JUPITER_FIELDS()) {
                    currSnapshot[columnName] = row[columnName];
                }
            }
        }

        if (!NYTUtils::IsNodeNull(fastBanNode)) {
            currSnapshot[F_IS_SEARCHABLE] = false;
            currSnapshot[F_HTTP_CODE] = fastBanNode[F_HTTP_CODE];
            currSnapshot[F_URL_STATUS] = NJupiter::EAcceptanceUrlForWebMasterSimpleStatus::AUFWSS_HTTP_ERROR;
            currSnapshot[F_LAST_ACCESS] = fastBanNode[F_LAST_ACCESS];
        }

        if (!spreadRows.empty()) {
            const NYT::TNode &row = spreadRows.rbegin()->second;
            if (row.HasKey(F_HTTP_CODE)) {
                currSnapshot[F_SPREAD_HTTP_CODE] = row[F_HTTP_CODE];
            }

            if (row.HasKey(F_LAST_ACCESS)) {
                currSnapshot[F_SPREAD_LAST_ACCESS] = row[F_LAST_ACCESS];
            }

            if (row.HasKey(F_MIME_TYPE)) {
                currSnapshot[F_SPREAD_MIME_TYPE] = row[F_MIME_TYPE];
            }
        }

        THashSet<TString> changedColumns;
        for (const auto &obj : currSnapshot.AsMap()) {
            const TString &column = obj.first;
            if (obj.second != prevSnapshot[column]) {
                changedColumns.insert(column);
            }
        }

        if (MarkDescriptionChanged && description.IsString()) {  // WMC-5446
            changedColumns.insert(F_DESCRIPTION);
        }

        for (const auto &obj : prevSnapshot.AsMap()) { //copy spread data from previous snapshot to current
            const TString &column = obj.first;
            NYT::TNode &currColumn = currSnapshot[column];
            const NYT::TNode &prevColumn = obj.second;
            if (NYTUtils::IsNodeNull(currColumn) && !NYTUtils::IsNodeNull(prevColumn)) {
                if (SPREAD_FIELDS().contains(column)) {
                    currSnapshot[column] = obj.second;
                } else {
                    changedColumns.insert(column);
                }
            }
            if (currColumn.GetType() == NYT::TNode::Undefined) {
                currColumn = nullNode;
            }
        }

        //currRow[F_USER_ID] = userId;
        output->AddRow(currSnapshot, OUTPUT_TABLENO_SNAPSHOT);

        if (!changedColumns.empty()) {
            NYT::TNode updateRow = currSnapshot;
            for (const auto &obj : currSnapshot.AsMap()) {
                const TString &column = obj.first;
                if (!SKIP_FIELDS().contains(column)) {
                    updateRow[CHANGED_PREFIX + column] = changedColumns.contains(column);
                }
            }

            NYT::TNode prevSpreadHttpCode = nullNode;
            NYT::TNode prevSpreadLastAccess = nullNode;
            NYT::TNode prevSpreadMimeType = nullNode;

            if (prevSnapshot.HasKey(F_SPREAD_HTTP_CODE) && prevSnapshot[F_SPREAD_HTTP_CODE].GetType() != NYT::TNode::Undefined) {
                prevSpreadHttpCode = prevSnapshot[F_SPREAD_HTTP_CODE];
            }

            if (prevSnapshot.HasKey(F_SPREAD_LAST_ACCESS) && prevSnapshot[F_SPREAD_LAST_ACCESS].GetType() != NYT::TNode::Undefined) {
                prevSpreadLastAccess = prevSnapshot[F_SPREAD_LAST_ACCESS];
            }

            if (prevSnapshot.HasKey(F_SPREAD_MIME_TYPE) && prevSnapshot[F_SPREAD_MIME_TYPE].GetType() != NYT::TNode::Undefined) {
                prevSpreadMimeType = prevSnapshot[F_SPREAD_MIME_TYPE];
            }

            updateRow[F_PREV_SPREAD_HTTP_CODE] = prevSpreadHttpCode;
            updateRow[F_PREV_SPREAD_LAST_ACCESS] = prevSpreadLastAccess;
            updateRow[F_PREV_SPREAD_MIME_TYPE] = prevSpreadMimeType;

            output->AddRow(updateRow, OUTPUT_TABLENO_UPDATES);
        }
    }

public:
    time_t JupiterTimestamp;
    TDeque<TImportantUrlsConfig> InputConfig;
    bool MarkDescriptionChanged; // WMC-5446
};

REGISTER_REDUCER(TImportantUrlsReducer)

bool IsTableEmpty(NYT::IClientBasePtr client, const TString &table) {
    NYTUtils::TTableInfo info;
    if (!NYTUtils::GetTableInfo(client, table, info)) {
        return true;
    }

    return info.RecordCount == 0;
}

bool IsUpdateHitThreshold(const TString &key, float value) {
    //https://solomon.yandex-team.ru/?cluster=webmaster_performance_production&project=webmaster&service=webmaster_quality&l.host=cluster&l.sensor=ImportantUrls_Share_Changed_*&graph=auto&stack=false&norm=false&downsampling=off&checks=&b=31d&e=

    const static THashMap<TString, float> THRESHOLDS = {
        {"_Changed_BeautyUrl",          0.030}, //0.465
        {"_Changed_Description",        0.015}, //0.807
        {"_Changed_HttpCode",           0.035}, //0.465
        {"_Changed_IsFake",             0.030}, //0.465
        {"_Changed_IsIndexed",          0.041}, //0.465
        {"_Changed_IsSearchable",       0.041}, //0.465
        {"_Changed_MainHost",           0.026}, //0.465
        {"_Changed_MainMirrorHost",     0.035}, //0.465
        {"_Changed_MainPath",           0.030}, //0.465
        {"_Changed_MimeType",           0.026}, //0.465
        {"_Changed_RedirTarget",        0.010}, //0.465
        {"_Changed_RelCanonicalTarget", 0.020}, //0.465
        {"_Changed_SpreadHttpCode",     0.020}, //0.018
        {"_Changed_SpreadMimeType",     0.021}, //0.037
        {"_Changed_Title",              0.016}, //0.863
        {"_Changed_UrlStatus",          0.045}, //0.465
    };

    if (THRESHOLDS.contains(key) && value > THRESHOLDS.at(key)) {
        return true;
    }

    return false;
}

void MonitorChangesShare(NYT::IClientBasePtr client, const TString &tableName) {
    const TConfig &config = TConfig::CInstance();

    THashMap<TString, size_t> counters;
    auto reader = client->CreateTableReader<NYT::TNode>(tableName);
    NYTUtils::TTableInfo tableInfo;
    NYTUtils::GetTableInfo(client, config.TABLE_IMPORTANT_URLS_SOURCE_URLS_RAW, tableInfo);
    const size_t rows = tableInfo.RecordCount;
    for (; reader->IsValid(); reader->Next()) {
        const NYT::TNode &row = reader->GetRow();
        for (const auto &node : row.AsMap()) {
            if (node.first.Contains(CHANGED_PREFIX) && !node.second.IsNull() && node.second.AsBool()) {
                counters[node.first]++;
            }
        }
    }

    bool susp = false;
    NYT::TNode suspParams = NYT::TNode::CreateList();

    for (const auto &obj : counters) {
        const float share = static_cast<float>(obj.second) / static_cast<float>(rows);
        if (IsUpdateHitThreshold(obj.first, share)) {
            susp = true;
            suspParams.Add(NYT::TNode()
                (obj.first, share)
            );
        }
        MonitorPushImportantUrlsUpdateShare(config.MONITOR_PERFORMANCE_SUFFIX, obj.first, share);
    }

    if (susp) {
        NYTUtils::SetAttr(client, tableName, TAttrName::Suspicious, true);
        NYTUtils::SetAttr(client, tableName, TAttrName::SuspiciousParams, suspParams);
    }
}

static NYT::TRichYPath DebugPath(const TString &table) {
    NYT::TRichYPath path(table);
/*
    path.AddRange(NYT::TReadRange().Exact(NYT::TReadLimit().Key(NYT::TKey("https://lenta.ru"))));
    path.AddRange(NYT::TReadRange().Exact(NYT::TReadLimit().Key(NYT::TKey("http://www.pleer.ru"))));
    path.AddRange(NYT::TReadRange().Exact(NYT::TReadLimit().Key(NYT::TKey("http://odobri.ru"))));
    path.AddRange(NYT::TReadRange().Exact(NYT::TReadLimit().Key(NYT::TKey("http://www.vedomosti.ru"))));
    path.AddRange(NYT::TReadRange().Exact(NYT::TReadLimit().Key(NYT::TKey("http://www.bloknotov.ru"))));
*/
    return path;
}

int TaskUpdate(int, const char **) {
    LOG_INFO("Running TaskUpdate!");
    const TConfig &config = TConfig::CInstance();

    NYT::IClientPtr client = NYT::CreateClient(config.MR_SERVER_HOST);
    NYT::ITransactionPtr tx = client->StartTransaction();

    const TString jupiterTable = GetJupiterAcceptanceTable(tx);
    const TString contentTable = GetJupiterContentAttrsTable(tx);
    const time_t timestamp = Now().Seconds();
    const TString updatesTable = NYTUtils::JoinPath(config.TABLE_IMPORTANT_URLS_UPDATES, ToString(timestamp));

    NYT::TTableSchema preparedUrlsSchema = CreatePreparedUrlsSchema();
    NYT::TTableSchema snapshotSchema = CreateSnapshotSchema(tx);
    NYT::TTableSchema updateSchema = CreateUpdateSchema(snapshotSchema);

    NYTUtils::CreatePath(tx, config.TABLE_IMPORTANT_URLS_UPDATES);

    TOpRunner(tx)
        .InputNode(config.TABLE_IMPORTANT_URLS_SOURCE_URLS_RAW)
        .OutputNode(NYT::TRichYPath(config.TABLE_IMPORTANT_URLS_SOURCE_URLS_PREPARED).Schema(preparedUrlsSchema))
        .Map(new TImportantUrlsPathConvertMapper)
        .SortBy(F_HOST, F_PATH, F_TABLE_TIMESTAMP)
        .Sort(config.TABLE_IMPORTANT_URLS_SOURCE_URLS_PREPARED)
    ;

    TDeque<NYTUtils::TTableInfo> jupiterSources;
    TDeque<NYTUtils::TTableInfo> spreadSources;
    TDeque<NYTUtils::TTableInfo> callistoSources;

    NYTUtils::GetTableList(tx, config.TABLE_IMPORTANT_URLS_SOURCE_JUPITER, jupiterSources);
    NYTUtils::GetTableList(tx, config.TABLE_IMPORTANT_URLS_SOURCE_SPREAD, spreadSources);
    NYTUtils::GetTableList(tx, config.TABLE_IMPORTANT_URLS_SOURCE_CALLISTO, callistoSources);

    std::sort(jupiterSources.begin(), jupiterSources.end(),
        [] (const NWebmaster::NYTUtils::TTableInfo &lhs, const NWebmaster::NYTUtils::TTableInfo &rhs) -> bool {
            return lhs.Name > rhs.Name;
        }
    );

    TOpRunner runner(tx);
    TOpRunner sortRunner(tx); //FIX: remove

    TDeque<TImportantUrlsConfig> inputConfig;

    inputConfig.push_back(TImportantUrlsConfig(TImportantUrlsConfig::E_TABLE_IMPORTANT_URLS));
    runner.InputNode(config.TABLE_IMPORTANT_URLS_SOURCE_URLS_PREPARED);

    inputConfig.push_back(TImportantUrlsConfig(TImportantUrlsConfig::E_TABLE_FAST_BAN));
    runner.InputNode(TCommonYTConfigIndexing::CInstance().TABLE_SOURCE_JUPITER_FASTBAN);

    inputConfig.push_back(TImportantUrlsConfig(TImportantUrlsConfig::E_TABLE_TURBO_PAGE));
    runner.InputNode(TCommonYTConfigIndexing::CInstance().TABLE_SOURCE_TURBO_PAGE);

    size_t jupiterSourceTables = 0;
    TString selectedJupiterSource;
    for (const NYTUtils::TTableInfo &table : jupiterSources) {
        if (IsTableEmpty(tx, table.Name)) {
            LOG_WARN("important_urls, empty table: %s", table.Name.data());
            continue;
        }

        const TString name = NYTUtils::GetTableName(table.Name);
        inputConfig.push_back(TImportantUrlsConfig(TImportantUrlsConfig::E_TABLE_JUPITER_BASE, FromString(name)));
        runner.InputNode(table.Name);
        jupiterSourceTables++;

        if (!NYTUtils::IsTableSorted(tx, table.Name)) {
            sortRunner.SortBy(F_HOST, F_PATH).Sort(table.Name, ASYNC_CTX0);
        }

        selectedJupiterSource = table.Name;
        break;
    }

    for (const NYTUtils::TTableInfo &table : spreadSources) {
        if (IsTableEmpty(tx, table.Name)) {
            LOG_WARN("important_urls, empty table: %s", table.Name.data());
            continue;
        }

        const TString name = NYTUtils::GetTableName(table.Name);
        inputConfig.push_back(TImportantUrlsConfig(TImportantUrlsConfig::E_TABLE_SPREAD_EXPORT, FromString(name)));
        runner.InputNode(table.Name);

        if (!NYTUtils::IsTableSorted(tx, table.Name)) {
            sortRunner.SortBy(F_HOST, F_PATH).Sort(table.Name, ASYNC_CTX0);
        }
    }

    for (const NYTUtils::TTableInfo &table : callistoSources){
        if (IsTableEmpty(tx, table.Name)) {
            LOG_WARN("important_urls, empty table: %s", table.Name.data());
            continue;
        }

        const TString name = NYTUtils::GetTableName(table.Name);
        inputConfig.push_back(TImportantUrlsConfig(TImportantUrlsConfig::E_TABLE_CALLISTO_BASE, FromString(name)));
        runner.InputNode(table.Name);

        if (!NYTUtils::IsTableSorted(tx, table.Name)) {
            sortRunner.SortBy(F_HOST, F_PATH).Sort(table.Name, ASYNC_CTX0);
        }
    }

    sortRunner.Wait(ASYNC_CTX0);

    if (jupiterSourceTables == 0) {
        LOG_INFO("important_urls, there is no Jupiter source tables");
        return 0;
    }

    inputConfig.push_back(TImportantUrlsConfig(TImportantUrlsConfig::E_TABLE_CONTENT_ATTRS));
    runner.InputNode(DebugPath(contentTable));

    if (tx->Exists(config.TABLE_IMPORTANT_URLS_SNAPSHOT)) {
        const TImportantUrlsConfig::ETableType tableType = TImportantUrlsConfig::E_TABLE_SNAPSHOT;
        //TOpRunner(tx).Copy(config.TABLE_IMPORTANT_URLS_SNAPSHOT, config.TABLE_IMPORTANT_URLS_SNAPSHOT + "." + ToString(Now().Seconds()));
        inputConfig.push_back(TImportantUrlsConfig(tableType));
        runner.InputNode(config.TABLE_IMPORTANT_URLS_SNAPSHOT);
    }

    /** WMC-5446: передаём в редьюсер информацию о том, помечать ли все ненулевые description'ы
      * как обновлённые. Чтобы сделать это ровно один раз, заводим в //home/webmaster/{env}/important_urls/ табличку-флаг.
      */
    const bool markDescriptionChanged = !tx->Exists(config.TABLE_IMPORTANT_URLS_FLAG);
    if (markDescriptionChanged) {
        Cout << "Marking all non-null description as changed" << Endl;
        tx->Create(config.TABLE_IMPORTANT_URLS_FLAG, NYT::NT_TABLE, NYT::TCreateOptions());
    }

    runner
        .OutputNode(NYT::TRichYPath(config.TABLE_IMPORTANT_URLS_SNAPSHOT).Schema(snapshotSchema))
        .OutputNode(NYT::TRichYPath(updatesTable).Schema(updateSchema))
        .MemoryLimit(4_GBs)
        .ReduceBy(F_HOST, F_PATH)
        .Reduce(new TImportantUrlsReducer(inputConfig, markDescriptionChanged))
        .SortBy(F_HOST, F_PATH)
        .Sort(config.TABLE_IMPORTANT_URLS_SNAPSHOT)
    ;

    NYTUtils::SetAttr(tx, updatesTable, TAttrName::Imported, true);

    NYTUtils::TTableInfo urlsUpdatedTableInfo;
    NYTUtils::TTableInfo urlsProcessedTableInfo;
    NYTUtils::GetTableInfo(tx, updatesTable, urlsUpdatedTableInfo);
    NYTUtils::GetTableInfo(tx, config.TABLE_IMPORTANT_URLS_SOURCE_URLS_PREPARED, urlsProcessedTableInfo);
    size_t urlsUpdated = urlsUpdatedTableInfo.RecordCount;
    size_t urlsProcessed = urlsProcessedTableInfo.RecordCount;

    if (urlsUpdated == 0) {
        tx->Remove(updatesTable);
    } else {
        TOpRunner(tx)
            .SortBy(F_HOST, F_PATH)
            .Sort(updatesTable)
        ;
        MonitorChangesShare(tx, updatesTable);
    }

    for (const NYTUtils::TTableInfo &table : jupiterSources) {
        if (selectedJupiterSource != table.Name) {
            tx->Remove(table.Name);
        }
    }

    for (const NYTUtils::TTableInfo &table : spreadSources) {
        tx->Remove(table.Name);
    }

    for(const NYTUtils::TTableInfo &table : callistoSources){
        tx->Remove(table.Name);
    }

    tx->Commit();

    MonitorPushImportantUrlsUpdated(config.MONITOR_PERFORMANCE_SUFFIX, urlsUpdated);
    MonitorPushImportantUrlsProcessed(config.MONITOR_PERFORMANCE_SUFFIX, urlsProcessed);

    return 0;
}

int TaskUpdateDev(int, const char **) {
    const TConfig &config = TConfig::CInstance();

    NYT::IClientPtr client = NYT::CreateClient(config.MR_SERVER_HOST);
    NYT::ITransactionPtr tx = client->StartTransaction();

    TDeque<NYTUtils::TTableInfo> tables;
    NYTUtils::GetTableList(tx, config.TABLE_IMPORTANT_URLS_UPDATES, tables);

    if (!tables.empty()) {
        MonitorChangesShare(tx, tables[0].Name);
    }

    tx->Commit();

    return 0;
}

} //namespace NImportantUrls
} //namespace NWebmaster
