#include "rtyserver.h"
#include "rtyserver_diag.h"
#include "startup_journal.h"

#include <saas/library/mapping/mapping.h>

#include <saas/rtyserver/factors/factors_config.h>
#include <saas/rtyserver/unistat_signals/signals.h>
#include <saas/rtyserver/common/metrics.h>
#include <saas/rtyserver/common/search_area_modifier.h>
#include <saas/rtyserver/config/common_indexers_config.h>
#include <saas/rtyserver/config/config.h>
#include <saas/rtyserver/config/logger_config.h>
#include <saas/rtyserver/config/monitor_config.h>
#include <saas/rtyserver/config/searcher_config.h>
#include <saas/rtyserver/config/sys_env.h>
#include <saas/rtyserver/indexer_core/index_component_storage.h>
#include <saas/rtyserver/regular/dead_docs_clear.h>
#include <saas/rtyserver/regular/dead_indexes_clear.h>
#include <saas/util/logging/exception_process.h>

#include <ysite/yandex/common/global_indexer_flags.h>

#include <kernel/index_mapping/index_mapping.h>
#include <kernel/factor_slices/factor_domain.h>

#include <library/cpp/logger/global/global.h>
#include <library/cpp/watchdog/lib/factory.h>
#include <library/cpp/watchdog/port_check/watchdog.h>

#include <util/folder/dirut.h>
#include <util/generic/ptr.h>
#include <util/stream/file.h>
#include <util/system/mutex.h>
#include <util/random/random.h>

#define TIMED_ACTION(Action, message) \
{\
    ui64 startTime = millisec();\
    StatusProgress += TString(message) + "...";\
    INFO_LOG << "Start " << message << Endl;\
    Action;\
    INFO_LOG << "Finished " << message << Endl;\
    StatusProgress += TString("Ok (") + ToString(millisec() - startTime) + " ms)\n";\
}

TRTYServer::TRTYServer(const TRTYServerConfig& config)
    : Config(config)
    , SearchEnginesManager(new TSearchEnginesManager(Config))
    , DocumentModifier(Config, SearchEnginesManager.Get())
    , DefaultKeyPrefix(config.GetSearcherConfig())
    , BaseSearchServer(new TBaseSearchServer(*SearchEnginesManager, Config.GetBaseSearcherConfig()))
    , SearchServer(new TMainSearchServer(*SearchEnginesManager, Config.GetSearcherConfig(), &DefaultKeyPrefix))
    , MetaSearchServer(new TMetaSearchNehServer(*SearchEnginesManager, Config.GetSearcherConfig(), Config.GetMetaSearcherConfig(), &DefaultKeyPrefix))
    , IndexStorage(TIndexStorage::Create(TIndexStorageConstructionContext(config)))
    , IndexMerger(*IndexStorage, config)
    , Status(tsStopped)
{
    InitializeBalloc();
    if (Config.GetSearcherConfig().Factors->IsInitialized()) {
        NFactorSlices::TFactorDomain::ForceSetGlobalUniverse(NFactorSlices::EFactorUniverse::SAAS);
    } else {
        NFactorSlices::TFactorDomain::ForceSetGlobalUniverse(NFactorSlices::EFactorUniverse::DEFAULT);
    }
    RunExecuted = false;
    SearchEnginesManager->Start();
    ResetMappedFiles();

    NRTYServer::EnableGlobalMapping(Config.IsReadOnly);

    if (Config.GetSysEnv().Io.contains("BatchRead")) {
        EnableThrottledReading(Config.GetSysEnv().Io.find("BatchRead")->second);
    }

    TSaasRTYServerSignals::BuildSignals(Config.GetRealmListConfig().GetRealmConfigNames(), Config.GetAddSignalTags());

    //Disk
    const TString& serviceName = Config.GetDaemonConfig().GetController().DMOptions.Service;
    IndexerServers.push_back(MakeHolder<TIndexerServer>(*IndexStorage, DocumentModifier, config.GetRealmListConfig().GetMainRealmConfig().GetIndexerConfigDisk(), serviceName));
    TIndexerManager* diskManager = new TIndexerManager(config.GetRealmListConfig().GetMainRealmConfig().GetIndexerConfigDisk(), serviceName, *IndexStorage, DocumentModifier, IndexerServers.back()->GetDocumentsQueue());
    IndexerServers.back()->RegisterManager(diskManager);
    IndexRepair.Reset(new TIndexRepair(*IndexStorage, *diskManager, config, DocumentModifier));
    DiagState.Reset(new TRTYServerDiagState());
    RegisterGlobalMessageProcessor(this);
}

TRTYServer::~TRTYServer() {
    if (!RunExecuted) {
        SearchEnginesManager->Stop();
    }
    UnregisterGlobalMessageProcessor(this);
}

void TRTYServer::InitializeBalloc() const {
    NAllocSetup::SetSoftLimit(Config.GetSysEnv().BallocOptions.SoftLimit);
    NAllocSetup::SetHardLimit(Config.GetSysEnv().BallocOptions.HardLimit);
    NAllocSetup::SetAllocationThreshold(Config.GetSysEnv().BallocOptions.AllocationThreshold);
    NAllocSetup::SetSoftReclaimDivisor(Config.GetSysEnv().BallocOptions.SoftReclaimDivisor);
    NAllocSetup::SetAngryReclaimDivisor(Config.GetSysEnv().BallocOptions.AngryReclaimDivisor);
}

void TRTYServer::StartSearchServer() {
    SearchServer->Start();
    if (MetaSearchServer) {
        MetaSearchServer->Start();
        if (SearchEnginesManager) {
            // because SEM may be initialized before BaseSearchersServer->Start() (before neh/inproc subsystem),
            // the metasearch needs explicit prefetch here
            SearchEnginesManager->PrefetchAtStart();
        }
    }
    StartSearchWatchDog();
    TSaasRTYServerSignals::UpdateSearchDisabledStatus(false);
}

void TRTYServer::StopSearchServer() {
    StopSearchWatchDog();
    if (!!SearchServer && SearchServer->IsRunning()) {
        TIMED_ACTION(SearchServer->ShutdownAndWait(), "SearchServer stopping");
    }
    if (MetaSearchServer) {
        TIMED_ACTION(MetaSearchServer->Stop(), "MetaSearchServer stopping");
    }
    TSaasRTYServerSignals::UpdateSearchDisabledStatus(true);
}

bool TRTYServer::IsSearchServerRunning() {
    return SearchServer->IsRunning();
}

void TRTYServer::StartSearchWatchDog() {
    if (Config.GetMonitorConfig().Enabled) {
        TPortCheckWatchDogHandle::TCheckerSettings settings;
        settings.Port = Config.GetSearcherConfig().ServerOptions.Port;
        settings.LimitCheckFail = Config.GetMonitorConfig().LimitCheckFail;
        settings.Request = "ping";
        settings.CheckIntervalSeconds = Config.GetMonitorConfig().CheckIntervalSeconds;
        SearchPortChecker.Reset(Singleton<TWatchDogFactory>()->RegisterWatchDogHandle(new TPortCheckWatchDogHandle(settings)));
    }
}

void TRTYServer::StopSearchWatchDog() {
    SearchPortChecker.Destroy();
}

void TRTYServer::Run() {
    TRY
        RunExecuted = true;
        TGlobalIndexerFlags::SetNumbersStyleNoZeros(true);
        TGuard<TMutex> g(StopMutex);
        VERIFY_WITH_LOG(Status != tsActive, "Attemption to run started RTYServer was detected");
        Status = tsStarting;
        Start = Now();
        StatusProgress = "Init objects...";
        NOTICE_LOG << "Starting rtyserver" << Endl;
        //Initialization of RandomNumber ...
        RandomNumber<ui64>();
        //Initialization of RandomNumber finished
        if (Config.WatchdogOptionsFiles) {
            WatchdogOptions = MakeIntrusive<TWatchdogOptionsHandle>(Config.WatchdogOptionsFiles, TDuration::Seconds(5));
            WatchdogOptions->Enable();
            IndexMerger.SetWatchdog(WatchdogOptions);
        }
        StatusProgress += "OK\n";
        StatusProgress += "Init indexes...";
        Singleton<TSearchAreaModifier>()->RegisterModifier(*SearchEnginesManager.Get());
        TMergerEngine::RegisterMerger(IndexMerger);
        //TMergerEngine::Subscribe(*WatchdogOptions);
        IndexStorage->Init();
        StatusProgress += "OK\n";
        VERIFY_WITH_LOG(!!BaseSearchServer, "Base search server not created");
        BaseSearchServer->Start();
        VERIFY_WITH_LOG(!!SearchServer, "Search server not created");
        if (Config.GetSearcherConfig().AutoStartServer) {
            StartSearchServer();
        } else {
            DEBUG_LOG << "Search server has not been started automatically" << Endl;
        }

        TIMED_ACTION(IndexRepair->Start(), "Repairer starting");

        IndexRepair->Wait();

        StatusProgress += "Indexers starting...";
        if (Config.GetCommonIndexers().Enabled) {
            StartIndexers();
        }
        StatusProgress += "OK\n";

        TIMED_ACTION(IndexMerger.Start(), "Merger starting");
        DeadDocsCleaner.Reset(Singleton<TWatchDogFactory>()->RegisterWatchDogHandle(new TDeadDocsCleanerWatchDogHandle(Config.DeadDocsClearIntervalSeconds)));
        DeadIndexesCleaner.Reset(Singleton<TWatchDogFactory>()->RegisterWatchDogHandle(new TDeadIndexesCleanerWatchDogHandle(Config.DeadIndexesClearIntervalSeconds)));
        Status = tsActive;
        NOTICE_LOG << StatusProgress << Endl;
        StatusProgress = "";
        if (Config.GetLoggerConfig().JournalDir.size()) {
            StartupJournal.Reset(new TStartupJournal(Config.GetLoggerConfig().JournalDir, ToString<ui16>(Config.GetSearcherConfig().ServerOptions.Port)));
            ProcessStartupJournal();
        }
    CATCH_AND_RETHROW("while rtyserver starting");
}

void TRTYServer::ProcessStartupJournal() const {
    INFO_LOG << StartupJournal->Report() << Endl;

    if (StartupJournal->IsAfterCrash()) {
        const auto& crashCodes = Config.GetMonitorConfig().CrashCodes;
        const TStartupJournal::TStopEvent& shutdownInfo = StartupJournal->GetShutdownInfo();

        if (crashCodes.find(shutdownInfo.Code) != crashCodes.end() && StartupJournal->GetCrashCount() >= Config.GetMonitorConfig().CrashRetries) {
            SendGlobalMessage<TSystemStatusMessage>(shutdownInfo.Message, TSystemStatusMessage::ESystemStatus::ssUnknownError);
        }
    }
}

void TRTYServer::StopIndexers(TString typeIndexer, bool rigidStop) {
    NOTICE_LOG << "Stopping " << typeIndexer << " indexers" << Endl;
    for (unsigned i = 0; i < IndexerServers.size(); ++i) {
        if (IndexerServers[i]->Config.GetType() == typeIndexer)
            IndexerServers[i]->StopIndexerServer(rigidStop);
    }
    for (unsigned i = 0; i < IndexerServers.size(); ++i) {
        if (IndexerServers[i]->Config.GetType() == typeIndexer) {
            IndexerServers[i]->Wait(rigidStop);
            NOTICE_LOG << "Indexer server thread exited" << Endl;
        }
    }
    if (Y_UNLIKELY(Config.ExternalServiceLogic)) {
        Config.ExternalServiceLogic->FinishIndexation();
    }
}

void TRTYServer::StartIndexers() {
    if (Y_UNLIKELY(Config.ExternalServiceLogic)) {
        CHECK_WITH_LOG(IndexerServers.size() == 1);
        Config.ExternalServiceLogic->StartIndexation(*IndexerServers.back());
    }
    for (unsigned i = 0; i < IndexerServers.size(); ++i)
        VERIFY_WITH_LOG(IndexerServers[i]->StartIndexerServer(), "Cannot run indexer server");

    for (unsigned i = 0; i < IndexerServers.size(); ++i)
        VERIFY_WITH_LOG(IndexerServers[i]->WaitReady(), "Indexer server didn't start");
}

void TRTYServer::Stop(ui32 rigidStopLevel, const TCgiParameters* cgiParams) {
    Y_UNUSED(cgiParams);
    TRY
        TGuard<TMutex> g(StopMutex);
        if(Status != tsActive) {
           FATAL_LOG << "Try to stop not active rtyserver";
           return;
        }
        Status = tsStopping;
        NOTICE_LOG << "Stopping rtyserver" << Endl;
        StatusProgress.clear();
        TIMED_ACTION(IndexRepair->Stop(rigidStopLevel >= rslHIGH), "Repair stopping");

        StopSearchServer();

        if (Config.GetCommonIndexers().Enabled) {
            TIMED_ACTION(StopIndexers("disk", rigidStopLevel >= rslHIGH), "Disk indexers stopping");
        }
        NMessenger::TGlobalMediator::CheckList("MemoryIndexer");
        NMessenger::TGlobalMediator::CheckList("Indexer");
        DeadIndexesCleaner.Destroy();
        TIMED_ACTION(IndexMerger.Stop(rigidStopLevel >= rslLOW), "Merger stopping");
        DeadDocsCleaner.Destroy();
        if (!!BaseSearchServer) {
            BaseSearchServer->Stop();
        }
        IndexStorage->StopSearching();
        if (!!SearchEnginesManager)
            TIMED_ACTION(SearchEnginesManager->Stop(), "SearchEnginesManager stopping");
        TIndexComponentsStorage::Instance().ReleaseComponents();
        IndexStorage->Stop();
        Status = tsStopped;
        DEBUG_LOG << StatusProgress << Endl;
        StatusProgress = "";
        NOTICE_LOG << "All indexer server threads are finished" << Endl;
        NOTICE_LOG << "TRTYserver::Stop finished" << Endl;
        if (StartupJournal.Get()) {
            StartupJournal->ServerExits();
        }
        if (WatchdogOptions) {
            WatchdogOptions->Disable();
        }
        return;
    CATCH("while rtyserver stopping");
    StatusProgress += "\nStop fail";
    ERROR_LOG << "TRTYserver::Stop fail" << Endl;
    Status = tsFailOnStop;
}

void TRTYServer::WaitEmptyQueues() const {
    bool isEmpty;
    do {
        isEmpty = true;
        for (unsigned i = 0; i < IndexerServers.size(); ++i) {
            isEmpty &= IndexerServers[i]->GetDocumentsQueue().IsEmpty();
        }
        if (!isEmpty)
            Sleep(TDuration::MilliSeconds(500));
    } while ((Status == tsActive) && (!isEmpty));
}

bool TRTYServer::Process(IMessage* message) {
    TMessageSearchServerControl* messSearchServerControl = dynamic_cast<TMessageSearchServerControl*>(message);
    if (messSearchServerControl) {
        if (Status != tsActive) {
            messSearchServerControl->SetRunning(false);
        } else {
            TGuard<TMutex> g(StopMutex);
            if (messSearchServerControl->ShouldStart() && !IsSearchServerRunning()) {
                StartSearchServer();
                SearchBanTimestamp = TInstant::Zero();
                DiagState->OnEnableSearchCommand();
            } else if (messSearchServerControl->ShouldStop() && IsSearchServerRunning()) {
                StopSearchServer();
                SearchBanTimestamp = TInstant::Now();
            }
            messSearchServerControl->SetRunning(IsSearchServerRunning());
        }
        return true;
    }

    TMessageCollectServerInfo* messInfo = dynamic_cast<TMessageCollectServerInfo*>(message);
    if (messInfo) {
        messInfo->SetStatus(ToString(Status) + (StatusProgress.empty() ? TString() : ("\n" + StatusProgress + "\n")));
        messInfo->SetUptime(Now() - Start);
        messInfo->SetSearchEnabled(IsSearchServerRunning());
        {
            const TAtomicSharedPtr<NRTYFactors::TConfig> factorsConfig = Config.GetSearcherConfig().Factors;
            if (!!factorsConfig) {
                messInfo->SetFactorsCount(factorsConfig->GetFactorCount());
                messInfo->SetFactorsConfig(factorsConfig->GetJsonConfig());
                messInfo->SetFactorsUsage(factorsConfig->GetUnusedFactorsReport());
            }
        }
        return true;
    }

    TMessageGetUsedFactors* messFactors = dynamic_cast<TMessageGetUsedFactors*>(message);
    if (messFactors) {
        const TAtomicSharedPtr<NRTYFactors::TConfig> factorsConfig = Config.GetSearcherConfig().Factors;
        if (!!factorsConfig) {
            messFactors->SetFactorsConfig(factorsConfig->GetJsonConfig());
            messFactors->SetFactorsUsage(factorsConfig->GetUnusedFactorsReport());
        }
        return true;
    }

    TMessageGetServerHealth *messHealth = dynamic_cast<TMessageGetServerHealth *>(message);
    if (messHealth) {
        if (StartupJournal.Get()) {
            messHealth->CrashCount = StartupJournal->GetCrashCount();
        }
        messHealth->SearchServerRunning = IsSearchServerRunning();
        messHealth->BanTimestamp = SearchBanTimestamp.Seconds();
        messHealth->RecoveryStage = DiagState->GetRecoveryStage(IsRepairing(), messHealth->SearchServerRunning);
        return true;
    }

    TMessageItsOverride *messOverride = dynamic_cast<TMessageItsOverride*>(message);
    if (messOverride && WatchdogOptions) {
        WatchdogOptions->AsUpdater().Override(messOverride->Key, messOverride->Value);
        return true;
    }

    return false;
}
