#include "common.h"
#include "subscriber.h"
#include "processor.h"

#include <saas/api/export/jsondocarray.pb.h>
#include <saas/api/clientapi.h>

#include <saas/library/daemon_base/metrics/messages.h>

#include <saas/indexerproxy/adapters/json_to_rty/json_to_rty_adapter.h>
#include <saas/indexerproxy/adapters/proto_adapter/proto_adapter.h>
#include <saas/indexerproxy/common/messages.h>
#include <saas/indexerproxy/context/inproc.h>
#include <saas/indexerproxy/export/config.pb.h>
#include <saas/indexerproxy/logging/tskv_helpers.h>
#include <saas/indexerproxy/server/sender_executor.h>
#include <saas/util/logging/tskv_log.h>
#include <library/cpp/logger/global/global.h>

#include <library/cpp/http/misc/httpcodes.h>

#include <library/cpp/digest/md5/md5.h>
#include <util/network/socket.h>

#include <google/protobuf/text_format.h>

using namespace NKiwi;

namespace {
    const TString DefaultHost = "localhost";
    const ui16 DefaultAcceptedCode = 200;

    const TString ServiceMetricsPrefix = "export_service_";
    const TString ExportMetricsPrefix = "export_";

    class TTskvSyslogBackend: public TFileLogBackend {
    public:
        TTskvSyslogBackend(const TString& path)
            : TFileLogBackend(path)
        {}

        void WriteData(const TLogRecord& rec) override {
            size_t len = rec.Len;
            if (len && rec.Data[len - 1] == '\n')
                --len;

            const TStringBuf data(rec.Data, len);
            NUtil::TTSKVRecord mess("saas-ipr-log");
            mess.AddIsoEventTime().ForceAdd("kind", "ip-kwsyslog");
            TskvConvertKiwiLegacy(mess, data);

            TString line = mess.ToString();
            line.append('\n');

            TLogRecord endlined(rec.Priority, line.data(), line.size());
            TFileLogBackend::WriteData(endlined);
        }
    };


    class TAutoEventGuard {
    private:
        TSystemEvent& E;
    public:
        TAutoEventGuard(TSystemEvent& e)
            : E(e)
        {}
        ~TAutoEventGuard() {
            E.Signal();
        }
    };
}

namespace NSaas {
    struct TIncomingExportCtx {
        ui64 Id = 0;
        ui32 KeyType = 0;
        TString Name;
        TString RemoteAddr;
        NKiwi::NExport::TClusterWriter* CookieWriter = nullptr;
    };

    class TExportedDocument: public TInternalDocument {
    private:
        NUtil::TRepliesStorage& RepliesStorage;
        NKiwi::NExport::TClusterWriter* CookieWriter;
        TServiceMetricPtr Metric;
        TAutoEventGuard EventGuard;
        TAutoMetricGuard MetricGuard;
        const TString KiwiKey;
        const EAckMode AckMode;

        ui8 CookieKeyType;
        TString CookieKey;
        TString CookieData;

    public:
        TExportedDocument(const TIncomingExportCtx& exportCtx, const TIncomingDocumentCtx& documentCtx, TSystemEvent& e, NUtil::TRepliesStorage& repliesStorage)
            : TInternalDocument(documentCtx.Service, documentCtx.Adapter, documentCtx.Data)
            , RepliesStorage(repliesStorage)
            , CookieWriter(exportCtx.CookieWriter)
            , Metric(documentCtx.Metric)
            , EventGuard(e)
            , MetricGuard(Metric)
            , KiwiKey(documentCtx.ExportRecord.GetKey())
            , AckMode(documentCtx.AckMode)
        {
            NKiwi::NExport::TCookie cookie(documentCtx.ExportRecord, exportCtx.KeyType);
            CookieKeyType = cookie.KeyType;
            CookieKey  = cookie.Key;
            CookieData = cookie.Data;

            Source = "export";
            Origin = exportCtx.Name;
            RemoteAddr  = exportCtx.RemoteAddr;
            QueryString = "id=" + ToString(exportCtx.Id);
            ScriptName  = "/export";
        }

    protected:
        void DoReply(ui32 codeReply, const TString& message, const TString& statusMessage) override {
            DEBUG_LOG << "action=exported_doc_reply;code=" << codeReply << ";message=" << message << ";status=" << statusMessage << Endl;
            const bool success = codeReply == DefaultAcceptedCode;
            if (AckMode == EAckMode::Always ||
                (AckMode == EAckMode::Default && success) ||
                (AckMode == EAckMode::Inverse && !success))
            {
                Ack();
            }

            if (IsUserError(codeReply)) {
                auto record = MakeSimpleShared<NUtil::TRepliesStorage::TRecord>();
                record->Document = KiwiKey;
                record->TimeProcessed = Seconds();
                record->Reply = message;
                RepliesStorage.AddDocument(GetServiceName(),record);
            }
        }

    private:
        void Ack() const {
            DEBUG_LOG << "action=send_ack;keytype=" << CookieKeyType << ";cookie_url=" << CookieKey << Endl;
            CookieWriter->AckObject(CookieKeyType, CookieKey, CookieData);
        }
    };
}

NSaas::TKiwiExport::TKiwiExport(const TKiwiExportConfig& config, TTaskExecutor& executor)
    : Config(config)
    , TaskExecutor(executor)
    , ServiceMetrics(&GetGlobalMetrics(), ServiceMetricsPrefix)
    , ExportMetrics(&GetGlobalMetrics(), ExportMetricsPrefix)
    , RepliesStorage(10)
{
    if (!Config.Enabled) {
        return;
    }

    if (!!Config.LogFile) {
        Log.ResetBackend(THolder(new TFileLogBackend(Config.LogFile)));
    }

    NExport::TConfig exportConfig;
    NExport::TConfig::TSubscriberSrv& subscriberConfig = exportConfig.SubscriberSrv;
    subscriberConfig.Defaults();
    subscriberConfig.Port = config.Port;
    subscriberConfig.MaxInFlight = Config.MaxInFlight;
    subscriberConfig.MaxInFlightBySize = Config.MaxInBytes;
    subscriberConfig.NumWorkers = Config.Threads;

    NKiwi::TKwCliConf cliConf;
    cliConf.User = Config.User;
    cliConf.Workers = Config.Threads;
    cliConf.MaxInFlight = Config.MaxInFlight;
    cliConf.MaxInBytes = Config.MaxInBytes;
    cliConf.MaxTimeout = Config.MaxTimeout;
    cliConf.Retries = Config.ResendAttempts;

    MonitoringCounters.Reset(new NExport::TMonCounters);
    MonitoringCounters->Program = NExport::TConfig::Subscriber;
    Server.Reset(new NExport::TSubscriberServer(subscriberConfig, MonitoringCounters.Get()));
    CookieWriter.Reset(new NExport::TCookieWriter(cliConf, MonitoringCounters.Get()));
    if (Config.VerboseLogging && !!Config.LogFile) {
        NKiwi::KwLogInit("saas-export-subscriber", TSysLogBackend::TSYSLOG_LOCAL0, true, true, true, TLOG_DEBUG);
        SysLogInstance().ResetBackend(THolder(new TTskvSyslogBackend(Config.LogFile + "-kwsyslog")));
    }
    Subscriber.Reset(new TSubscriber(*this, *CookieWriter));
    Server->SetHandler(Subscriber.Get());

    NSaas::TTupleParsingOptions tupleOptions;
    if (const TString& file = Config.TupleParsingOptions) {
        using google::protobuf::TextFormat;
        VERIFY_WITH_LOG(NFs::Exists(file), "TupleParsingOptions file %s does not exist", file.data());
        VERIFY_WITH_LOG(TextFormat::ParseFromString(TIFStream(file).ReadAll(), &tupleOptions), "Cannot parse text protobuf %s", file.data());
    }
    ExportTypeTuple   = tupleOptions.GetExportTypeTuple();
    DefaultExportType = tupleOptions.GetDefaultExportType();

    UnknownProcessor = NSaas::CreateDefaultExportProcessor();
    for (auto&& exportType : tupleOptions.GetExportType()) {
        THolder<IExportProcessor> processor(NSaas::CreateExportProcessor(exportType));
        VERIFY_WITH_LOG(processor, "Processor %s does not exist", exportType.GetProcessor().data());
        Processors[exportType.GetName()] = processor.Release();
    }

    RegisterGlobalMessageProcessor(this);
}

NSaas::TKiwiExport::~TKiwiExport() {
    if (!Config.Enabled) {
        return;
    }

    UnregisterGlobalMessageProcessor(this);
}

void NSaas::TKiwiExport::Start() {
    if (!Config.Enabled) {
        INFO_LOG << "KiWi export disabled" << Endl;
        return;
    }
    INFO_LOG << "KiWi export subscriber starting... " << Endl;
    Server->Start();
    INFO_LOG << "KiWi export subscriber starting... OK" << Endl;
}

void NSaas::TKiwiExport::Stop() {
    if (!Config.Enabled) {
        INFO_LOG << "KiWi export has not been started" << Endl;
        return;
    }
    INFO_LOG << "KiWi export subscriber stopping... " << Endl;
    Server->Stop();
    INFO_LOG << "KiWi export subscriber stopping... OK" << Endl;
}

NSaas::TSubscriber::TSubscriber(TKiwiExport& exporter, NKiwi::NExport::TCookieWriter& cookieWriter)
    : Exporter(exporter)
    , CookieWriter(cookieWriter)
{}

#define EXPORT_LOG GetLog()

void NSaas::TSubscriber::DoHandle(TAutoPtr<TBusExportMessage> msg) {
    if (!msg) {
        NUtil::TTSKVRecord mess("saas-ipr-log");
        EXPORT_LOG << mess.AddIsoEventTime().ForceAdd("kind", "ip-subscriber").ForceAdd("status", "fail").ToString();
        return;
    }

    NSaas::TIncomingExportCtx exportCtx;
    exportCtx.Id   = msg->GetHeader()->Id;
    exportCtx.Name = msg->GetExportName();
    exportCtx.RemoteAddr = msg->Record.GetHenAddr().GetHost();
    exportCtx.KeyType = msg->Record.GetKeyType();

    TServiceMetricPtr metric = Exporter.GetExportMetrics().Get(exportCtx.Name);

    try {
        TAutoMetricGuard guard(metric);
        exportCtx.CookieWriter = CookieWriter.GetWriter(msg.Get());

        for (google::protobuf::RepeatedPtrField<NKwExport::TExportRecord>::const_iterator i = msg->Record.GetRecords().begin(), e = msg->Record.GetRecords().end(); i != e; ++i) {
            NUtil::TTSKVRecord mess("saas-ipr-log");
            EXPORT_LOG << mess.AddIsoEventTime().ForceAdd("kind", "ip-export").ForceAdd("origin", exportCtx.Name)
                    .ForceAdd("key", i->GetKey()).ForceAdd("size", i->GetData().size())
                    .ForceAdd("request_id", exportCtx.Id).ForceAdd("keytype", i->GetKeyType()).ToString() << Endl;

            Exporter.ProcessRecord(exportCtx, *i);
        }
    } catch (...) {
        NUtil::TTSKVRecord mess("saas-ipr-log");
        EXPORT_LOG << mess.AddIsoEventTime().ForceAdd("kind", "ip-subscriber").ForceAdd("status", "fail")
                .ForceAdd("origin", exportCtx.Name).ForceAdd("request_id", exportCtx.Id)
                .ForceAdd("expl", CurrentExceptionMessage()).ToString() << Endl;
    }
}

TLog& NSaas::TSubscriber::GetLog() {
    return Exporter.GetLog();
}

void NSaas::TKiwiExport::ProcessRecord(const TIncomingExportCtx& ctx, const NKwExport::TExportRecord& record) const {
    try {
        NKiwi::TKiwiObject object(record.GetData());

        const TString& type = GetTuple<TString>(object, ExportTypeTuple, DefaultExportType);
        auto p = Processors.find(type);
        VERIFY_WITH_LOG(p == Processors.end() || p->second, "null export processor");
        NSaas::TIncomingDocumentCtxPack docs = (p != Processors.end())
            ? p->second->Process(record)
            : UnknownProcessor->Process(record);
        for (auto&& dctx : docs) {
            IndexDocument(ctx, dctx);
        }
    } catch (const yexception& e) {
        NUtil::TTSKVRecord mess("saas-ipr-log");
        EXPORT_LOG << mess.AddIsoEventTime().ForceAdd("kind", "ip-export").ForceAdd("status", "fail")
                .ForceAdd("origin", ctx.Name).ForceAdd("request_id", ctx.Id)
                .ForceAdd("key", record.GetKey()).ForceAdd("expl", e.what()).ToString() << Endl;
    }
}

#undef EXPORT_LOG

TString NSaas::TKiwiExport::Name() const {
    return "KiwiExport";
}

bool NSaas::TKiwiExport::Process(IMessage* message_) {
    if (TCollectMetricsMessage* message = dynamic_cast<TCollectMetricsMessage*>(message_)) {
        if (!Config.Enabled) {
            message->GetOutputAccess() << "export disabled" << Endl;
            return true;
        }
        MonitoringCounters->OutputText(message->GetOutputAccess(), nullptr, false);
        return true;
    }
    if (TMessageCollectIndexerProxyInfo* message = dynamic_cast<TMessageCollectIndexerProxyInfo*>(message_)) {
        message->ExportReplies = RepliesStorage.GetReport();
        return true;
    }
    return false;
}

void NSaas::TKiwiExport::IndexDocument(const TIncomingExportCtx& exportCtx, TIncomingDocumentCtx documentCtx) const {
    documentCtx.Metric = ServiceMetrics.Get(documentCtx.Service);
    while (documentCtx.Metric->GetInFlightCount() > Config.MaxInFlight) {
        DocIndexedEvent.WaitI();
    }
    auto document = MakeHolder<NSaas::TExportedDocument>(exportCtx, documentCtx, DocIndexedEvent, RepliesStorage);
    TaskExecutor.AddContext(document.Release());
}
