#include "docstream.h"
#include "messages.h"

#include <saas/api/action.h>
#include <saas/rtyserver/config/common_indexers_config.h>
#include <saas/rtyserver/config/config.h>
#include <saas/rtyserver/common/common_messages.h>
#include <saas/rtyserver/docfetcher/library/types.h>
#include <saas/util/logging/replies.h>

#include <saas/library/indexer_client/client.h>

#include <library/cpp/logger/global/global.h>

namespace {
    const ui64 ErrorsCount = 10;

    void LogActionDetails(NUtil::TTSKVRecord& l, NFusion::TActionPtr action, const TString& service) {
        if (!service.empty()) {
            l.ForceAdd("service", service);
        }
        if (action) {
            const auto& message = action->ToProtobuf();
            NSaas::TActionDebugView view = NSaas::GetActionDetails(message);
            l.ForceAdd("action", view.Name); //only name
            l.ForceAdd("key", view.Key);
            l.Add("keyprefix", view.KeyPrefix);
            if (message.HasMessageId())
                l.ForceAdd("id", view.MessageId);
            for (const auto& loggingProperty: message.GetLoggingProperties()) {
                l.ForceAdd(loggingProperty.GetName(), loggingProperty.GetValue());
            }
        }
    }
}

namespace NFusion {
    struct TShardFilter {
        explicit TShardFilter(const TShardFilterOptions& options)
            : Options(options)
            , ShardDispatcherContext(Options.ShardType, Options.KpsShift)
            , ShardDispatcher(ShardDispatcherContext)
        {
        }

        bool InsideShardRange(const NSaas::TAction& action) const {
            auto shardIndex = ShardDispatcher.GetShard(action.ToProtobuf());
            return Options.ShardMin <= shardIndex && shardIndex <= Options.ShardMax;
        }

    private:
        const TShardFilterOptions Options;
        const NSaas::TShardsDispatcher::TContext ShardDispatcherContext;
        const NSaas::TShardsDispatcher ShardDispatcher;
    };
}

NFusion::TDocStream::TDocStream(
    const TDocFetcherModule& owner, const TDocStreamConfig& config, const TRTYServerConfig& globalConfig, TLog& log)
    : TBaseStream(owner, config, log, "DfDocStrm")
    , RTYServerConfig(globalConfig)
    , DocStreamConfig(config)
    , Errors(MakeHolder<NUtil::TRepliesStorage>(ErrorsCount))
    , BaseMetrics(config.Name)
{
    if (config.ShardFilter.IsEnabled()) {
        ShardFilter = MakeHolder<TShardFilter>(config.ShardFilter);
    }
}

NFusion::TDocStream::~TDocStream() = default;

void NFusion::TDocStream::OnStreamStart() {
    if (DocStreamConfig.IndexingHost && DocStreamConfig.IndexingPort) {
        IndexingClient = MakeHolder<NFusion::TRemoteIndexingClient>(
            DocStreamConfig.IndexingHost,
            DocStreamConfig.IndexingPort,
            DocStreamConfig.ClientOptions);
    } else {
        IndexingClient = MakeHolder<NFusion::TInternalIndexingClient>(RTYServerConfig.GetCommonIndexers().ServerOptions.Port, DocStreamConfig.ClientOptions);
    }
    OnDocStreamStart();

    const ui64 delay = GetDelay().Seconds();
    if (delay) { //(yrum) zero delay at stream start may mean that we have no data
        SignalsBase()->SetReceiveTimestamp(Now().Seconds() - delay);
    }
    if (delay > DocStreamConfig.SearchOpenThreshold) {
        NOTICE_LOG << "Stream " << DocStreamConfig.Name << " has found excessive delay " << delay << Endl;
        DisableSearch();
    } else if (delay <= DocStreamConfig.SearchOpenThreshold && SearchDisabledByStream()) {
        NOTICE_LOG << "Stream " << DocStreamConfig.Name << " has acceptable delay " << delay << Endl;
        EnableSearch();
    }
    StartTimestamp = Now();
}

void NFusion::TDocStream::OnStreamBeforeStop() {
    IndexingClient->Shutdown();
}

void NFusion::TDocStream::OnStreamStop() {
    OnDocStreamStop();
    IndexingClient.Destroy();
}

NFusion::IThreadLoop::TTimeToSleep NFusion::TDocStream::DoStreamIteration() {
    const TInstant now = TInstant::Now();
    TryToEnableSearch(now);
    TMap<TString, TString> properties;
    bool skipSleepOnError = false;
    if (auto doc = GetDoc(properties, skipSleepOnError)) {
        doc = ProcessDoc(doc, properties, now);
        if (doc) {
            if (ShardFilter && !ShardFilter->InsideShardRange(*doc)) {
                return TTimeToSleep::Zero();
            }
            OnAcquired(doc);
            DEBUG_LOG << "Acquired document " << doc->GetActionDescription() << Endl;
            IndexDoc(doc);
        }
        return TTimeToSleep::Zero();
    } else {
        TTimeToSleep timeToSleep = DocStreamConfig.TimeToSleep;
        if (IsExhausted()) {
            OnExhausted(timeToSleep);
        }
        return skipSleepOnError ? TTimeToSleep::Zero() : timeToSleep;
    }
}

void NFusion::TDocStream::TryToEnableSearch(const TInstant& now) {
    if (LastDocumentTimestamp == TInstant::Zero()) {
        return;
    }
    const TDuration age = now - LastDocumentTimestamp;

    if (SearchDisabledByStream()) {
        ui64 threshold = static_cast<i64>(now.Seconds()) >= AtomicGet(PessimizeEndTimestamp)
            ? DocStreamConfig.SearchOpenThreshold
            : DocStreamConfig.SearchOpenThresholdPessimistic;
        if (age.Seconds() < threshold) {
            NOTICE_LOG << "Stream " << BaseConfig.Name << " delay " << age
                       << " is less than SearchOpenThreshold " << threshold << Endl;
            EnableSearch();
        }
    }
}

NFusion::IThreadLoop::TTimeToSleep NFusion::TDocStream::OnException() {
    BaseMetrics.Exceptions.Inc();
    ERROR_LOG << "An exception occurred in stream " << BaseConfig.Name << ": " << CurrentExceptionMessage() << Endl;
    return TTimeToSleep::Seconds(1);
}

void NFusion::TDocStream::OnAcquired(NFusion::TActionPtr doc) {
    BaseMetrics.Acquired.Inc();
    SignalsBase()->ProcessIncoming(doc);
    switch (DocStreamConfig.RealtimeOverride) {
        case ERealtimeOverrideMode::DoNotOverride:
            break;
        case ERealtimeOverrideMode::ForceFalse:
            doc->GetDocument().SetRealtime(false);
            break;
        case ERealtimeOverrideMode::ForceTrue:
            doc->GetDocument().SetRealtime(true);
            break;
        default:
            Y_VERIFY(false, "Unknown ERealtimeOverrideMode");
    }
}

bool NFusion::TDocStream::IsExhausted() const {
    const TDuration idle = Now() - (LastIncoming ? LastIncoming : StartTimestamp);
    return (LastIncoming >= LastExhausted) && (idle > DocStreamConfig.MaxIdleTime);
}

void NFusion::TDocStream::OnExhausted(TTimeToSleep& /*timeToSleep*/) {
    LastExhausted = TInstant::Now();
    SignalsBase()->ProcessExhausted();
    if (SearchDisabledByStream()) {
        NOTICE_LOG << "Stream " << DocStreamConfig.Name << " enabling search on exhausted" << Endl;
        EnableSearch();
    }
    if (DocStreamConfig.ReopenIndexesOnExhaustion) {
        NOTICE_LOG << "Stream " << DocStreamConfig.Name << " reopening indexers on exhausted" << Endl;
        ReopenIndexers();
    }
}

void NFusion::TDocStream::OnReply(const NRTYServer::TReply& reply, TActionPtr action) {
    SignalsBase()->ProcessIndexed(action, reply);

    auto status = static_cast<NRTYServer::TReply::TRTYStatus>(reply.GetStatus());
    if (status == NRTYServer::TReply::OK) {
        return;
    }

    NUtil::TTSKVRecord l("saas-df-log");
    l.AddIsoEventTime();
    l.ForceAdd("stream", BaseConfig.Name);
    l.ForceAdd("stream_id", BaseConfig.StreamId);
    LogActionDetails(l, action, RTYServerConfig.GetDaemonConfig().GetController().DMOptions.Service);
    l.ForceAdd("kind", "df-index");
    l.Add("status", NRTYServer::TReply_TRTYStatus_Name(status));
    l.Add("expl", reply.GetStatusMessage());
    Log << l.ToString() << Endl;

    auto error = MakeSimpleShared<NUtil::TRepliesStorage::TRecord>();
    error->Document = action ? action->GetActionDescription() : "";
    error->Reply = reply.GetStatusMessage();
    error->TimeProcessed = Seconds();
    error->Timestamp = action ? action->GetDocument().GetTimestamp() : 0;
    Errors->AddDocument("Errors", error);
}

void NFusion::TDocStream::IndexDoc(NFusion::TActionPtr doc) {
    auto doc_ = doc;
    std::function<void(const NRTYServer::TReply&)> handler = [doc_, this](const NRTYServer::TReply& reply) {
        OnReply(reply, doc_);
    };

    CHECK_WITH_LOG(IndexingClient);
    IndexingClient->Send(doc, handler, !DocStreamConfig.FastIndexWhenNoSearch || !SearchDisabledByStream());
}

bool NFusion::TDocStream::Process(IMessage* message_) {
    if (auto message = message_->As<TMessageGetDocfetcherStatus>()) {
        CHECK_WITH_LOG(IndexingClient);
        message->DocfetcherPresent = true;
        message->DocsIndexing += IndexingClient->GetInFlight();
        message->QueueSize += IndexingClient->GetQueueSize();
        if (Errors) {
            message->Replies.InsertValue(BaseConfig.Name, Errors->GetReport());
        }
        ReportStatus(message->Status);
        return true;
    }
    if (auto message = message_->As<TMessagePessimizeDocfetcherBan>()) {
        if (message->Pessimize) {
            AtomicSet(PessimizeEndTimestamp, Seconds() + DocStreamConfig.SearchOpenPessimizationDelaySeconds);
        } else {
            AtomicSet(PessimizeEndTimestamp, 0);
        }
        return true;
    }
    return TBaseStream::Process(message_);
}

NFusion::TActionPtr NFusion::TDocStream::ProcessDoc(NFusion::TActionPtr action, const TMap<TString, TString>& propsToLog, const TInstant& now) {
    LastIncoming = Now();

    const TInstant receiveTimestamp = action->GetReceiveTimestamp();
    const TInstant docTimestamp = TInstant::Seconds(action->GetDocument().GetTimestamp());
    const i64 version = action->GetDocument().GetVersion();

    const bool isUpdate = action->GetProtobufActionType() == NRTYServer::TMessage::DEPRECATED__UPDATE_DOCUMENT;
    const bool updateWithZeroTs = isUpdate && docTimestamp == TInstant::Zero();

    const TInstant timestamp = DocStreamConfig.ReceiveDurationAsDocAge ? receiveTimestamp : docTimestamp;

    const ui64 receiveDuration = (now - receiveTimestamp).Seconds();
    const ui64 age = (now - timestamp).Seconds();

    const bool realtime = (age < DocStreamConfig.MemIndexAgeSec) && (receiveDuration < DocStreamConfig.MemoryIndexDistAgeThreshold);
    const bool expired = !updateWithZeroTs && (age >= DocStreamConfig.MaxDocAgeToKeepSec);

    NUtil::TTSKVRecord record("saas-df-log");
    record.AddIsoEventTime();
    record.ForceAdd("stream", BaseConfig.Name);
    record.ForceAdd("stream_id", BaseConfig.StreamId);
    LogActionDetails(record, action, RTYServerConfig.GetDaemonConfig().GetController().DMOptions.Service);
    record.ForceAdd("age", age);
    record.ForceAdd("dtimestamp", docTimestamp.Seconds());
    record.ForceAdd("kind", "df");
    record.ForceAdd("version", version);
    record.ForceAdd("docstream", action->GetDocument().GetStreamId());
    record.ForceAdd("receive_duration", receiveDuration);
    record.ForceAdd("receive_timestamp", receiveTimestamp.Seconds());
    record.Add("non_realtime", !realtime);
    record.Add("expired", expired);
    for (auto& property : propsToLog) {
        record.Add(property.first, property.second);
    }
    Log << record.ToString() << Endl;

    if (!updateWithZeroTs) {
        LastDocumentTimestamp = timestamp;
    }

    SignalsBase()->ProcessAttrs(action);

    if (expired) {
        SignalsBase()->ProcessExpired(action);
        return nullptr;
    }
    if (!realtime) {
        action->GetDocument().SetRealtime(realtime);
    }
    return action;
}
