#include <util/draft/date.h>
#include <util/generic/set.h>
#include <util/thread/pool.h>

#include <robot/library/yt/static/command.h>
#include <yql/library/embedded/yql_embedded.h>
#undef LOG_DEBUG
#undef LOG_INFO
#undef LOG_ERROR

#include <wmconsole/version3/processors/user_sessions/conf/config.h>
#include <wmconsole/version3/processors/user_sessions/protos/monitoring.pb.h>
#include <wmconsole/version3/processors/user_sessions/protos/user_sessions.pb.h>
#include <wmconsole/version3/processors/user_sessions/library/monitor.h>

#include "monitoring.h"

namespace NWebmaster {
namespace NMonitoring {

const char *FORMAT = "%Y-%m-%d";

using namespace NJupiter;

void MoveMetrikaRefToMain(NYT::IClientBasePtr clientDst, const TString &table) {
    NYT::IClientPtr clientSrc = NYT::CreateClient(TConfig::CInstance().MR_SERVER_HOST_QUERIES);
    NYT::ITransactionPtr txDst = clientDst->StartTransaction();
    auto reader = TTable<NMonitoring::NProto::TReferenceMetrika>(clientSrc, table).GetReader();
    auto writer = TTable<NMonitoring::NProto::TReferenceMetrika>(txDst, table)
        .AsSortedOutput({"Host", "CounterID", "IsMobile", "IsPad"})
        .GetWriter()
    ;
    for (; reader->IsValid(); reader->Next()) {
        writer->AddRow(reader->GetRow());
    }
    writer->Finish();
    txDst->Commit();
    clientSrc->Remove(table);
}

THolder<NYql::NEmbedded::IOperation> RunYQL(const TString &sql, const TString &operationTitle) {
    NYql::NEmbedded::TOperationFactoryOptions factoryOptions;
    auto factory = NYql::NEmbedded::MakeOperationFactory(factoryOptions);
    NYql::NEmbedded::TOperationOptions operationOptions;
    operationOptions.Title = operationTitle;
    operationOptions.SyntaxVersion = 1;
    auto operation = factory->Run(sql, operationOptions);
    //LOG_INFO("%s", TString(operation->YsonResult()).c_str());
    //LOG_INFO("%s", TString(operation->Plan()).c_str());
    return operation;
}

TStringBuf ServerAlias(const TStringBuf server) {
    return server.Before('.');
}

void UpdateMetrikaReference(NYT::IClientBasePtr clientMain, const TString &inputTablePublic, const TString &inputTablePrivate, const TString &outputTable) {
    TString SQL = R"(
        use ${YT};
        pragma yt.Pool = 'robot-webmaster';

        insert into `${OUTPUT_TABLE}`
            with truncate
        select
            CounterID,
            Host,
            IsMobile,
            IsPad,
            sum(Sign) as Visits,
            SUM((CAST(IsBounce == true AS Int32))*Sign) AS BounceCount,
            FromYandex
        from concat(`${INPUT_TABLE1}`, `${INPUT_TABLE2}`)
        where Host is not null
        group by
            String::ToLower(Url::HostNameToPunycode(Url::GetSchemeHost(StartURL))) as Host,
            CounterID,
            coalesce(IsMobile, False) as IsMobile,
            coalesce(IsTablet, False) as IsPad,
            coalesce(SearchEngineID, 0) IN (2, 13, 181) as FromYandex
        order by Host, CounterID, IsMobile, IsPad
    )";

    SubstGlobal(SQL, "${YT}", ServerAlias(TConfig::CInstance().MR_SERVER_HOST_QUERIES));
    SubstGlobal(SQL, "${INPUT_TABLE1}", inputTablePublic);
    SubstGlobal(SQL, "${INPUT_TABLE2}", inputTablePrivate);
    SubstGlobal(SQL, "${OUTPUT_TABLE}", outputTable);
    RunYQL(SQL, "SQ reference monitoring: UpdateMetrikaReference (YQL)");
    MoveMetrikaRefToMain(clientMain, outputTable);
}

void UpdateMetrikaReference(const TDate &startDate, const TDate &endDate, TSet<TDate> &affectedDates) {
    NYT::IClientPtr clientMain = NYT::CreateClient(TConfig::CInstance().MR_SERVER_HOST_MAIN);
    NYTUtils::CreatePath(clientMain, TCommonYTConfigSQ::CInstance().TABLE_PARSED_USER_SESSIONS_REF_METRIKA_ROOT);

    THolder<IThreadPool> queue(CreateThreadPool(4));
    for (TDate curDate = startDate; curDate > endDate; --curDate) {
        const TString dateStr = curDate.ToStroka(FORMAT);
        const TString inputTablePublic = NYTUtils::JoinPath(TCommonYTConfigSQ::CInstance().TABLE_SOURCE_METRIKA_VISIT_LOG_ROOT, dateStr);
        const TString inputTablePrivate = NYTUtils::JoinPath(TCommonYTConfigSQ::CInstance().TABLE_SOURCE_METRIKA_VISIT_PRIVATE_LOG_ROOT, dateStr);
        const TString outputTable = NYTUtils::JoinPath(TCommonYTConfigSQ::CInstance().TABLE_PARSED_USER_SESSIONS_REF_METRIKA_ROOT, dateStr);
        const TString statsTable = NYTUtils::JoinPath(TCommonYTConfigSQ::CInstance().TABLE_PARSED_USER_SESSIONS_STATS_DAILY_ROOT, dateStr);

        if (clientMain->Exists(outputTable)) {
            continue;
        }

        if (!clientMain->Exists(statsTable)) {
            continue;
        }

        queue->SafeAddFunc([=, &affectedDates, &clientMain]() {
            try {
                LOG_INFO("monitoring, Metrika reference %s", dateStr.c_str());
                UpdateMetrikaReference(clientMain, inputTablePublic, inputTablePrivate, outputTable);
                affectedDates.insert(curDate);
                LOG_INFO("monitoring, Metrika reference %s - done", dateStr.c_str());
            } catch(yexception &e) {
                LOG_ERROR("monitoring, unable to process %s: %s", inputTablePublic.c_str(), e.what());
            }
        });
    }
    queue->Stop();
}

void UpdateStatistics(const TString &dateStrBegin, const TString &dateStrEnd) {
    TString SQL = R"(
        use ${YT};
        pragma yt.Pool = 'robot-webmaster';

        $date1 = '${DATE_STR1}';
        $date2 = '${DATE_STR2}';

        $ref_metrika_root = '${REF_METRIKA_ROOT}';
        $ref_user_sessions_root = '${REF_USER_SESSIONS_ROOT}';
        $ref_statistics = '${REF_STATISTICS}';

        $max_counters = (
            select Period, Host, max_by(CounterID, Visits) as MaxCounterID, FromYandex
            from range($ref_metrika_root, $date1, $date2)
            group by TableName() as Period, Host, FromYandex
            having max(Visits) > 100
        );

        $visits = (
            select Period, Host, CounterID, IsMobile, IsPad, sum(Visits) as Visits, sum(BounceCount) as BounceCount, FromYandex
            from range($ref_metrika_root, $date1, $date2)
            group by TableName() as Period, Host, IsMobile, IsPad, CounterID, FromYandex
        );

        $metrika = (
            select src.Period as Period, src.Host as Host, IsMobile, IsPad, Visits, BounceCount
            from $visits as src
            inner join $max_counters as flt
                on src.Period == flt.Period
                and src.Host == flt.Host
                and src.CounterID == flt.MaxCounterID
            where src.FromYandex
        );

        $top_user_sessions = (
            select Period, Host, sum(Clicks) as Clicks
            from range($ref_user_sessions_root, $date1, $date2)
            group by TableName() as Period, Host
            having sum(Clicks) > 100
        );

        $user_sessions = (
            select src.Period as Period, src.Host as Host, IsMobile, IsPad, src.Clicks as Clicks
            from (
                select Period, Host, IsMobile, IsPad, sum(Clicks) as Clicks
                from range($ref_user_sessions_root, $date1, $date2)
                group by TableName() as Period, Host, IsMobile, IsPad
            ) as src
            inner join $top_user_sessions as flt
                on src.Period == flt.Period
                and src.Host == flt.Host
        );

        $diff = (
            select
                m.Period as Period,
                m.Host as Host,
                m.IsMobile as IsMobile,
                m.IsPad as IsPad,
                Clicks,
                Visits,
                BounceCount,
                if (Visits == 0, 1.0, cast(Clicks - Visits as float) / cast(Visits as float)) as DiffAB,
                if (Clicks == 0, 1.0, cast(Visits - Clicks as float) / cast(Clicks as float)) as DiffBA
            from $metrika as m
            inner join $user_sessions as u
                on m.Period == u.Period
                and m.Host == u.Host
                and m.IsMobile = u.IsMobile
                and m.IsPad == u.IsPad
        );

        insert into $ref_statistics
            with truncate
        select
            Period,
            IsMobile,
            IsPad,
            some(MetrikaMedian) as MetrikaMedian,
            some(MetrikaP90) as MetrikaP90,
            some(MetrikaP98) as MetrikaP98
        from (
            select Period, IsMobile, IsPad, MetrikaMedian, MetrikaP90, MetrikaP98
            from $ref_statistics
            union all
            select Period, IsMobile, IsPad, median(DiffAB) as MetrikaMedian, percentile(DiffAB, 0.9) as MetrikaP90, percentile(DiffAB, 0.98) as MetrikaP98
            from $diff
            where Clicks > 100 or Visits > 100
            group by Period, IsMobile, IsPad
        )
        group by Period, IsMobile, IsPad
        order by Period, IsMobile, IsPad
    )";

    SubstGlobal(SQL, "${YT}", ServerAlias(TConfig::CInstance().MR_SERVER_HOST_MAIN));
    SubstGlobal(SQL, "${DATE_STR1}", dateStrBegin);
    SubstGlobal(SQL, "${DATE_STR2}", dateStrEnd);
    SubstGlobal(SQL, "${REF_METRIKA_ROOT}", TCommonYTConfigSQ::CInstance().TABLE_PARSED_USER_SESSIONS_REF_METRIKA_ROOT);
    SubstGlobal(SQL, "${REF_USER_SESSIONS_ROOT}", TCommonYTConfigSQ::CInstance().TABLE_PARSED_USER_SESSIONS_STATS_DAILY_ROOT);
    SubstGlobal(SQL, "${REF_STATISTICS}", TCommonYTConfigSQ::CInstance().TABLE_PARSED_USER_SESSIONS_REF_STATS);
    RunYQL(SQL, "SQ reference monitoring: UpdateStatistics (YQL)");
}

void UpdateMonitoring(const TString &dateStr1, const TString &dateStr2) {
    TMap<time_t, TMap<TString, double>> history;
    NYT::IClientPtr client = NYT::CreateClient(TConfig::CInstance().MR_SERVER_HOST_MAIN);
    auto reader = TTable<NProto::TStatistics>(client, TCommonYTConfigSQ::CInstance().TABLE_PARSED_USER_SESSIONS_REF_STATS).GetReader();
    for (; reader->IsValid(); reader->Next()) {
        const auto &row = reader->GetRow();
        if (row.GetPeriod() < dateStr1 || row.GetPeriod() > dateStr2) {
            continue;
        }
        const time_t timestamp = TDate(row.GetPeriod(), FORMAT).GetStart();
        const bool isDesktop = !row.GetIsMobile() && !row.GetIsPad();
        const bool isMobile = row.GetIsMobile() && !row.GetIsPad();
        const bool isPad = row.GetIsMobile() && row.GetIsPad();
        auto &record = history[timestamp];
        if (isDesktop) {
            record["DesktopMedian"] = row.GetMetrikaMedian();
            record["DesktopP90"] = row.GetMetrikaP90();
            record["DesktopP98"] = row.GetMetrikaP98();
        } else if (isMobile) {
            record["MobileMedian"] = row.GetMetrikaMedian();
            record["MobileP90"] = row.GetMetrikaP90();
            record["MobileP98"] = row.GetMetrikaP98();
        } if (isPad) {
            record["PadMedian"] = row.GetMetrikaMedian();
            record["PadP90"] = row.GetMetrikaP90();
            record["PadP98"] = row.GetMetrikaP98();
        }
    }

    MonitorPushHistory(TConfig::CInstance().MONITOR_PERFORMANCE_SUFFIX, history, "Queries_RefMetrika_");
}

void UpdateMetrikaReference() {
    //const TDate startDate = TDate("2019-05-06", FORMAT);
    //const TDate endDate = TDate("2019-01-01", FORMAT);
    const TDate startDate = TDate::Today() - 1;
    const TDate endDate = startDate - 30;

    TSet<TDate> affectedDates;
    UpdateMetrikaReference(startDate, endDate, affectedDates);

    if (!affectedDates.empty()) {
        const TString &dateStr1 = affectedDates.begin()->ToStroka(FORMAT);
        const TString &dateStr2 = affectedDates.rbegin()->ToStroka(FORMAT);
        UpdateStatistics(dateStr1, dateStr2);
        UpdateMonitoring(dateStr1, dateStr2);
    }

    //UpdateStatistics("2019-07-04", "2019-07-08");
    //UpdateMonitoring("2019-07-04", "2019-07-08");
}

} //namespace NMonitoring
} //namespace NWebmaster
