#include <util/draft/date.h>
#include <util/generic/set.h>
#include <util/thread/pool.h>

#include <robot/library/yt/static/command.h>
#include <robot/library/yt/static/table.h>
#include <yql/library/embedded/yql_embedded.h>
#include <kernel/urlnorm/normalize.h>

#undef LOG_DEBUG
#undef LOG_INFO
#undef LOG_ERROR

#include <wmconsole/version3/processors/siteservices/protos/sources.pb.h>
#include <wmconsole/version3/processors/siteservices/protos/services.pb.h>
#include <wmconsole/version3/processors/siteservices/conf/config.h>
#include <wmconsole/version3/wmcutil/yt/transfer_manager.h>

#include "import_util.h"
#include "task_about_news_hosts.h"

namespace NWebmaster {
using namespace NJupiter;


bool GetSites(THashMap<TString, NJson::TJsonValue> &newsSiteInfo) {
    try{
        const TString newsData = "https://proxy.sandbox.yandex-team.ru/last/NEWS_REPORT_DATA_PROD/agencies.json";
        const auto answer = HttpGet<TString>(newsData, 5000);
        LOG_INFO("Download completed");
        NJson::TJsonValue root;
        TStringStream json(answer);
        NJson::ReadJsonTree(&json, &root);
        THashMap<TString, NJson::TJsonValue> tmp;
        root.GetMap(&tmp);
        tmp["aid"].GetMap(&newsSiteInfo);
        return true;
    } catch (yexception &) {
        return false;
    }
}


int NewsHosts(int, const char **) {
    const auto &cfg = TConfig::CInstance();
    NYT::IClientPtr client = NYT::CreateClient(cfg.MR_SERVER_HOST_NEWS);

    NYTUtils::CreatePath(client, cfg.TABLE_SERVICES_PREP_ROOT);


    LOG_INFO("News, Start import from sanbox");
    TString tableTmp = cfg.TABLE_SERVICES_PREP_NEWS + "-tmp";
    NYT::ITransactionPtr tx = client->StartTransaction();
    auto writer = tx->CreateTableWriter<NProto::TPrepNews>(tableTmp);
    THashMap<TString, NJson::TJsonValue> newsSiteInfo;
    for(int i =0; i < 100; i++){
        if (GetSites(newsSiteInfo)){
            break;
        }
        if (i == 99){
            LOG_INFO("News, can't be dowloaded!");
            return 0;
        }
        sleep(1000);
    }

    NProto::TPrepNews prepNews;
    for (const auto &entry : newsSiteInfo) {
        const NJson::TJsonValue &info = entry.second;
        LOG_INFO("Site: %s, has_enabled_feeds:",info["site_url"].GetStringSafe().c_str());
        if (info["has_enabled_feeds"].GetIntegerSafe() > 0){
            prepNews.SetMascotOwner(GetOwner(info["site_url"].GetStringSafe()));
            writer->AddRow(prepNews);
        }
    }


    LOG_INFO("News, Done copy table");

    writer->Finish();

    TSortCmd<NProto::TPrepNews>(tx, TTable<NProto::TPrepNews>(tx, tableTmp))
            .By({"MascotOwner"})
            .OperationWeight(cfg.OPERATION_WEIGHT)
            .Do();

    LOG_INFO("News, Done sort table");

    auto readerTmp = tx->CreateTableReader<NProto::TPrepNews>(tableTmp);
    writer = tx->CreateTableWriter<NProto::TPrepNews>(cfg.TABLE_SERVICES_PREP_NEWS);
    TString prevOwner = "";
    for (; readerTmp->IsValid(); readerTmp->Next()) {
        const auto &row = readerTmp->GetRow();
        if (!prevOwner.equal(row.GetMascotOwner())) {
            TString normalized;
            if (NUrlNorm::NormalizeUrl(row.GetMascotOwner(), normalized)){
                prepNews.SetMascotOwner(GetOwner(normalized));
                writer->AddRow(prepNews);
            }
        }
        prevOwner = row.GetMascotOwner();
    }
    writer->Finish();

    TSortCmd<NProto::TPrepNews>(tx, TTable<NProto::TPrepNews>(tx, cfg.TABLE_SERVICES_PREP_NEWS))
            .By({"MascotOwner"})
            .OperationWeight(cfg.OPERATION_WEIGHT)
            .Do();

    tx->Commit();

    LOG_INFO("News, Import done");

    return 0;}
} //namespace NWebmaster
