#include "search_engines_manager.h"
#include "metasearch.h"

#include <saas/rtyserver/search/config/search_config_builder_meta.h>
#include <saas/rtyserver/search/common/handlers.h>
#include <saas/rtyserver/search/meta/replier.h>
#include <saas/rtyserver/search/prefetch/common.h>
#include <saas/rtyserver/search/source/search_replier.h>

#include <saas/library/daemon_base/metrics/messages.h>
#include <saas/library/daemon_base/threads/rty_pool.h>
#include <saas/library/daemon_base/unistat_signals/messages.h>
#include <saas/library/searchserver/protocol.h>
#include <saas/rtyserver/unistat_signals/signals.h>
#include <saas/rtyserver/common/common_messages.h>
#include <saas/rtyserver/common/stream_messages.h>
#include <saas/rtyserver/common/message_collect_server_info.h>
#include <saas/rtyserver/common/message_get_doc_info.h>
#include <saas/rtyserver/common/search_area_modifier.h>
#include <saas/rtyserver/config/searcher_config.h>
#include <saas/rtyserver/config/realm_config.h>
#include <saas/util/logging/messages.h>
#include <saas/util/queue.h>
#include <saas/util/types/interval.h>
#include <saas/util/logging/exception_process.h>
#include <saas/util/transaction.h>
#include <search/request/data/reqdata.h>

#include <kernel/urlid/doc_handle.h>

#include <library/cpp/mediator/messenger.h>

#include <util/system/hostname.h>
#include <util/system/info.h>

ISearchReplier::TPtr TSearchEnginesManager::TGuardedSearch::GetSearchReplier(const TString& customMetaSearch, ui64 defKps, IReplyContext::TPtr context) {
    CHECK_WITH_LOG(!!context);
    if (IsMetaSearch) {
        if (customMetaSearch == "first_found") {
            return new TCustomMetaSearchReplier(context, SEM.GetMetaSearchHandlers(), SEM.GetSearchersVector(SearchVersion), SEM.GetMainSearchServerMetrics(), SEM.GetConfig());
        } else {
            return new TRTYCommonSearchReplier(context, MetaSearch, SEM, defKps);
        }
    }

    if (!IndexController->GetCommonSearch()) {
        return new TCustomSearchReplier(context, SEM.GetBaseSearchHandlers(), IndexController, SEM.GetBaseSearchServerMetrics(), SEM.GetConfig());
    } else {
        return new TRTYCommonSearchReplier(context, IndexController->GetCommonSearch(), SEM, defKps);
    }
}

IEventLog& TSearchEnginesManager::GetEventLog() {
    return *EventLog.Get();
}

ui32 TSearchEnginesManager::DecodeDocId(const TDocHandle& docIdFromMessage, size_t indexId) const {
    if (indexId < GetSearchersVectorCurrent().size()) {
        return GetSearchersVectorCurrent()[indexId]->DecodeDocId(docIdFromMessage);
    }
    ythrow yexception() << "Incorrect id of index";
}

bool TSearchEnginesManager::DecodeDocId(const TDocHandle& docIdFromMessage, ui32& indexId, ui32& docId) const {
    indexId = docIdFromMessage.ClientNum();

    const TSearchersVector& searchers = GetSearchersVectorCurrent();
    if (indexId >= searchers.size())
        return false;

    docId = searchers[indexId]->DecodeDocId(docIdFromMessage.ClientDocId());
    return true;
}

ui32 TSearchEnginesManager::GetDocumentsCount(size_t indexId, bool withDeleted) const {
    const TSearchersVector& searchers = GetSearchersVectorCurrent();
    if (indexId >= searchers.size())
        return (ui32)-1;

    const auto& searcher = searchers[indexId];
    ui32 result = searcher->GetDocsCount();

    if (!withDeleted)
        result -= searcher->GetDeletedDocsCount();

    return result;
}

ui32 TSearchEnginesManager::GetTimestamp(size_t indexId, const TDocHandle& docIdFromMessage) const {
    const TSearchersVector& searchers = GetSearchersVectorCurrent();
    if (indexId >= searchers.size()) {
        return 0;
    }

    TRTYSearchSource::TPtr searcher = searchers[indexId];
    IIndexController::TPtr index = searcher->GetIndexController();
    const IDDKManager* ddk = index->GetDDKManager();
    if (!ddk) {
        return 0;
    }

    ui32 docId = searcher->DecodeDocId(docIdFromMessage);
    if (docId >= searcher->GetDocsCount()) {
        return 0;
    }
    return ddk->GetTimeLiveStart(docId);
}

namespace {
    bool SortByName(const TAtomicSharedPtr<TRTYSearchSource>& left, const TAtomicSharedPtr<TRTYSearchSource>& right) {
        if (left->IsMem() == right->IsMem()) {
            return left->GetDirName() < right->GetDirName();
        } else {
            return left->IsMem();
        }
    }

    bool SortByPriority(const TAtomicSharedPtr<TRTYSearchSource>& left, const TAtomicSharedPtr<TRTYSearchSource>& right) {
        ui32 leftP = left->GetSearchPriority();
        ui32 rightP = right->GetSearchPriority();
        if (leftP == rightP)
            return left->GetDirName() < right->GetDirName();
        return leftP > rightP;
    }
}

namespace NRTYServer {
    struct TTierInfo {
        TVector<TAtomicSharedPtr<TRTYSearchSource> > Sources;
        TInterval<double> PruneInterval;
        ui32 Intersections;

        bool IsEmpty() const {
            return !Sources.size();
        }

        void Merge(const TTierInfo& info) {
            PruneInterval.Join(info.PruneInterval);
            Sources.insert(Sources.end(), info.Sources.begin(), info.Sources.end());
        }

        ui64 GetDocsCount() const {
            ui64 result = 0;
            for (auto& i : Sources) {
                result += i->GetDocsCount();
            }
            return result;
        }

        ui64 GetDocsCount(const TInterval<double>& interval) const {
            ui64 result = 0;
            for (auto& i : Sources) {
                result += i->GetDocsCount(interval);
            }
            return result;
        }
    };

    struct TTiersInfo {
        TVector<TTierInfo> Tiers;

        ui32 GetTier(const TAtomicSharedPtr<TRTYSearchSource>& source) const {
            ui32 found = 0;
            ui32 result = 0;
            for (ui32 i = 0; i < Tiers.size(); ++i) {
                for (ui32 f = 0; f < Tiers[i].Sources.size(); ++f) {
                    if (Tiers[i].Sources[f] == source) {
                        result = i;
                        ++found;
                    }
                }
            }

            CHECK_WITH_LOG(found == 1);
            return result;
        }
    };

    THolder<TSearchHandlers> CreateSearchHandlers(const NRTYServer::TSearchServerOptions& config, const TString& serverType) {
        auto handlers = MakeHolder<TSearchHandlers>();
        handlers->MainHandler = TRTYSearchHandlers::CreateHandler(config.MainThreads, config.MainQueueSize, TRTYPools::TLoadtype::Primary, TString::Join(serverType, "MainHandler"));

        if (config.FactorThreads) {
            handlers->FactorsHandler = TRTYSearchHandlers::CreateHandler(config.FactorThreads, config.FactorQueueSize, TRTYPools::TLoadtype::Fetch, TString::Join(serverType, "FactorsHandler"));
        } else {
            handlers->FactorsHandler = handlers->MainHandler;
        }
        if (config.FetchThreads) {
            handlers->PassagesHandler = TRTYSearchHandlers::CreateHandler(config.FetchThreads, config.FetchQueueSize, TRTYPools::TLoadtype::Fetch, TString::Join(serverType, "FetchHandler"));
        } else {
            handlers->PassagesHandler = handlers->MainHandler;
        }
        if (config.LongReqsThreads) {
            handlers->LongReqsHandler = TRTYSearchHandlers::CreateHandler(config.LongReqsThreads, config.LongReqsQueueSize, TRTYPools::TLoadtype::Undefined, TString::Join(serverType, "LongReqsHandler"));
        } else {
            handlers->LongReqsHandler = handlers->MainHandler;
        }
        if (config.DbgReqsThreads) {
            handlers->DbgReqsHandler = TRTYSearchHandlers::CreateHandler(config.DbgReqsThreads, config.DbgReqsQueueSize, TRTYPools::TLoadtype::Undefined, TString::Join(serverType, "DbgReqsHandler"));
        } else {
            handlers->DbgReqsHandler = handlers->LongReqsHandler;
        }

        if (config.InfoThreads) { // be careful: we are creating InfoHandler via the different class (TSearchHandlers instead of the TRTYSearchHandlers)
            handlers->InfoHandler = TSearchHandlers::CreateHandler(config.InfoThreads, config.InfoQueueSize, /*elastic=*/false, TString::Join(serverType, "InfoHandler"));
        } else {
            handlers->InfoHandler = handlers->MainHandler;
        }

        handlers->ReAskHandler = handlers->LongReqsHandler;
        handlers->PreSearchReqsHandler = handlers->ReAskHandler;

        return handlers;
    }

    bool SortByIntersections(const TTierInfo& left, const TTierInfo& right) {
        return left.Intersections > right.Intersections;
    }

    bool SortByMin(const TTierInfo& left, const TTierInfo& right) {
        return left.PruneInterval.GetMin() > right.PruneInterval.GetMin();
    }
}

namespace {
    class TAreaSearchSourcesInfo {
    private:
        const ui32 SourcesCountLimit;
        const TString Path;
        TSet<TString> Sources;
        TMap<ui32, ui32> SpecPriorCount;
    public:
        TAreaSearchSourcesInfo(const TString& path, ui32 count)
            : SourcesCountLimit(count)
            , Path(path)
        {
        }

        bool Contain(const TRTYSearchSource& ss) const {
            return TFsPath(Path).IsContainerOf(ss.GetIndexController()->Directory().PathName());
        }

        bool Filter(const TRTYSearchSource& ss) const {
            return Sources.empty() || Sources.contains(ss.GetDirName());
        }

        void BuildData(TSearchEnginesManager::TSearchersVectorRef searchersVector) {
            Sources.clear();
            SpecPriorCount.clear();
            TMap<ui32, ui32> countByShards;
            TSearchersVector sizeSortVector = searchersVector;
            Sort(sizeSortVector.begin(), sizeSortVector.end(), SortByPriority);
            for (ui32 i = 0; i < sizeSortVector.size(); ++i) {
                DEBUG_LOG << Path << ":" << sizeSortVector[i]->GetIndexController()->Directory().PathName() << " checking ... (" << sizeSortVector[i]->GetIndexController()->GetSearchPriority() << ")" << Endl;

                if (!Contain(*sizeSortVector[i]))
                    continue;
                if (sizeSortVector[i]->IsMem() || !sizeSortVector[i]->GetIndexController()->IsSearchable())
                    continue;

                const ui32 shard = sizeSortVector[i]->GetIndexController()->GetShard();
                if (sizeSortVector[i]->GetIndexController()->GetSearchPriority())
                    ++SpecPriorCount[shard];
                if (countByShards[shard] < SourcesCountLimit) {
                    DEBUG_LOG << sizeSortVector[i]->GetDirName() << " checking ... OK" << Endl;
                    Sources.insert(sizeSortVector[i]->GetDirName());
                    ++countByShards[shard];
                }
            }
        }

        bool IsSpecialShard(ui32 shard) const {
            auto it = SpecPriorCount.find(shard);
            if (it != SpecPriorCount.end()) {
                return it->second > SourcesCountLimit;
            } else {
                return false;
            }
        }
    };
}

void TSearchEnginesManager::StartMetaSearch(TCommonSearchRef metaSearchP, TSearchersVectorRef searchersVector, ui32 searchVersion) {
    try {
        VERIFY_WITH_LOG(ActiveTransactions, "Incorrect usage of StartMetaSearch without transaction");
        CHECK_WITH_LOG(IsActive);

        Sort(searchersVector.begin(), searchersVector.end(), SortByName);
        if (searchersVector.size() == 0) {
            NOTICE_LOG << "Cannot start metasearch. No underlying searchers found." << Endl;
            return;
        }
        NOTICE_LOG << "Starting metasearch" << Endl;

        TAutoPtr<TSearchConfig> config(TSearchConfigBuilderMeta(Config).GetSearchConfig(TPathName(Config.GetSearcherConfig().GetPersistentSearchDir())));
        THolder<NRTYServer::TMetaSearch> metaSearch = MakeHolder<NRTYServer::TMetaSearch>(Config);

        if (!!EventLog) {
            Singleton<TSearchLogger>()->SetEventLog(EventLog);
        }
        config->CacheOptions.Main().UseCache = Config.GetSearcherConfig().CacheOptions.UseCache;

        NRTYServer::TTiersInfo tiers = BuildTiersInfo(searchersVector, Config.GetSearcherConfig().TierMinSize, Config.GetSearcherConfig().TiersOverlapPercent * 1.0 / 100.0);
        config->TierSignificantTable.push_back(false);
        config->ProtoCollection_.SetNTiers(tiers.Tiers.size());

        config->ProtoCollection_.SetEnableCacheForDbgRlv(Config.GetSearcherConfig().EnableCacheForDbgRlv);
        config->ProtoCollection_.SetReaskNotFullCacheAnswers(true);

        TVector<TAreaSearchSourcesInfo> searchSourcesInfo;
        for (auto&& [realmName, realmConfig] : Config.GetRealmListConfig().RealmsConfig) {
            if (realmConfig.StorageType == NRTYServer::EStorageType::Disk) {
                if (realmConfig.Type == NRTYServer::ERealm::Realtime) {
                    searchSourcesInfo.emplace_back(TAreaSearchSourcesInfo(realmConfig.RealmDirectory, EnableMemorySearch ? 1 : 0));
                } else {
                    searchSourcesInfo.emplace_back(TAreaSearchSourcesInfo(realmConfig.RealmDirectory, realmConfig.SearchersCountLimit));
                }
                searchSourcesInfo.back().BuildData(searchersVector);
            }
        }

        ui32 searchableDiskIndexesCount = 0;
        ui32 diskIndexesCount = 0;
        ui32 searchersCount = searchersVector.size();
        const auto activeDetect = [this, &diskIndexesCount, &searchableDiskIndexesCount, &searchersCount, &searchSourcesInfo]
            (const TAtomicSharedPtr<TRTYSearchSource> item) {
            diskIndexesCount += item->IsMem() ? 0 : 1;
            DEBUG_LOG << item->GetDirName() << " test for add ... " << Endl;
            if (!item->GetIndexController()->IsSearchable()) {
                item->SetActive(false, true);
                return true;
            }
            if (item->IsMem()) {
                if (!EnableMemorySearch && searchersCount > 1) {
                    item->SetActive(false, true);
                    return true;
                }
            } else {
                const TAreaSearchSourcesInfo* info = nullptr;
                for (const TAreaSearchSourcesInfo& searchSourceInfo : searchSourcesInfo) {
                    if (searchSourceInfo.Contain(*item)) {
                        info = &searchSourceInfo;
                        break;
                    }
                }
                CHECK_WITH_LOG(info);
                if (!info->Filter(*item)) {
                    const ui32 shard = item->GetIndexController()->GetShard();
                    item->SetActive(false, info->IsSpecialShard(shard));
                    return true;
                }
                ++searchableDiskIndexesCount;
            }
            DEBUG_LOG << item->GetDirName() << " test for add ... OK" << Endl;
            item->SetActive(true);
            return false;
        };

        ui32 movedCount = 0;
        for (ui32 i = 0; i < searchersVector.size() - movedCount;) {
            const TAtomicSharedPtr<TRTYSearchSource> item = searchersVector[i];
            if (activeDetect(item)) {
                searchersVector.erase(searchersVector.begin() + i);
                searchersVector.push_back(item);
                ++movedCount;
            } else {
                ++i;
            }
        }

        bool nActiveFound = false;
        const auto [hostName, basePort] = GetBaseHostAndPort();
        for (ui64 idSearcher = 0; idSearcher < searchersVector.size(); ++idSearcher) {
            TAtomicSharedPtr<TRTYSearchSource> iter = searchersVector[idSearcher];
            if (!iter->IsActive()) {
                nActiveFound = true;
                continue;
            } else {
                CHECK_WITH_LOG(!nActiveFound);
            }

            TSearchSource ss(++config->RealSearchSourcesCount);
            ss.ProtoConfig_.SetSearchScript(Sprintf("%s://%s:%u/%llu",
                /*scheme=*/InprocNehProtocol.data(),
                /*host=*/hostName.data(),
                /*port=*/basePort,
                /*searcher=*/(long long unsigned)(((ui64)idSearcher) | ((ui64)searchVersion << 32))
            ));
            // Do not change this.  It is used by Fusion rearrange rule to detect source type.
            ss.ProtoConfig_.SetServerDescr(iter->GetSourceDescr());
            ss.ProtoConfig_.SetServerGroup(iter->GetDirName());
            ss.Remote.CacheUpdateByTime = true;
            ss.Remote.Tier = tiers.GetTier(iter);

            if (Config.GetSearcherConfig().ReAskBaseSearches) {
                ss.ProtoConfig_.SetSearchScript(
                    ss.ProtoConfig_.GetSearchScript() + " " + ss.ProtoConfig_.GetSearchScript()
                );
            }
            ss.Remote.NoCache = !iter->GetCacheStatus();
            ss.Remote.CacheLifeTime = config->CacheOptions.Main().CacheLifeTime;
            ss.Remote.TwoStepQuery = config->TwoStepQuery;
            ss.Remote.EnableIpV6 = true;
            ss.Remote.ProtocolType = Config.GetSearcherConfig().ProtocolType;
            NOTICE_LOG << "Adding base search source for metasearch: " << ss.ProtoConfig_.GetSearchScript() << " - " << iter->GetDirName() << " " << (void*)iter->GetIndexController().Get() << Endl;
            config->SearchSources.push_back(ss);
        }
        SearchableIndexMetrics.SearchableDiskIndexNumber.Set(searchableDiskIndexesCount);
        SearchableIndexMetrics.IdleDiskIndexNumber.Set(diskIndexesCount - searchableDiskIndexesCount);

        metaSearch->SearchCache = CommonCache;
        metaSearch->FastCache = CommonFastCache;

        int searchOpenResult;
        do {
            searchOpenResult = metaSearch->SearchOpen(config);
            if (searchOpenResult != YXOK) {
                ERROR_LOG << "Cannot start metasearch!" << Endl;
                CHECK_WITH_LOG(Config.GetSearcherConfig().RetryStartMetasearch);
            }
        } while (searchOpenResult != YXOK);
        CHECK_WITH_LOG(metaSearch->IsSearching());
        NRTYServer::PrefetchCommon(Prefetcher.Get(), metaSearch.Get());

        metaSearchP = std::move(metaSearch);
    } catch (...) {
        ERROR_LOG << "On start meta search: " << CurrentExceptionMessage() << Endl;
        throw;
    }
}

std::pair<TString, ui16> TSearchEnginesManager::GetBaseHostAndPort() const {
    TString hostName = Config.GetSearcherConfig().LocalHostName ? Config.GetSearcherConfig().LocalHostName : HostName();
    return {hostName, Config.GetBaseSearcherConfig().Port};
}

NRTYServer::TTiersInfo TSearchEnginesManager::BuildTiersInfo(
    const TSearchersVector& searchersVector, ui64 minDocsCount, double crossPart) const {
    NRTYServer::TTiersInfo result;
    TVector<NRTYServer::TTierInfo>& tiers = result.Tiers;

    NRTYServer::TTierInfo specialTier;

    for (ui32 i = 0; i < searchersVector.size(); ++i) {
        NRTYServer::TTierInfo newTier;
        newTier.PruneInterval = searchersVector[i]->GetPruneInterval();
        newTier.Sources.push_back(searchersVector[i]);
        NOTICE_LOG << "PRUN_INTERVAL: " << searchersVector[i]->GetDirName() << " -> " << newTier.PruneInterval.ToString() << Endl;
        if (searchersVector[i]->IsMem() || searchersVector[i]->IsCustom()) {
            specialTier.Merge(newTier);
        } else {
            tiers.push_back(newTier);
        }
    }

    Sort(tiers.begin(), tiers.end(), NRTYServer::SortByMin);

    while (true) {
        ui32 countOnStart = tiers.size();
        for (ui32 i = 0; i < tiers.size(); ++i) {
            for (ui32 j = i + 1; j < tiers.size();) {
                TInterval<double> intersection;
                if (tiers[i].PruneInterval.Intersection(tiers[j].PruneInterval, intersection) &&
                    tiers[i].GetDocsCount(intersection) / tiers[i].GetDocsCount() >= crossPart)
                {
                    tiers[i].Merge(tiers[j]);
                    tiers.erase(tiers.begin() + j);
                    j = i + 1;
                } else {
                    ++j;
                }
            }
        }
        if (countOnStart == tiers.size())
            break;
    }

    for (ui32 i = 0; i + 1 < tiers.size();) {
        if (i >= Config.GetSearcherConfig().TiersCount - 1 || tiers[i].GetDocsCount() < minDocsCount) {
            tiers[i].Merge(tiers[i + 1]);
            tiers.erase(tiers.begin() + i + 1);
        } else {
            ++i;
        }
    }

    if (tiers.size() > 1 && tiers.back().GetDocsCount() < minDocsCount) {
        tiers[tiers.size() - 2].Merge(tiers.back());
        tiers.erase(tiers.begin() + tiers.size() - 1);
    }

    if (tiers.size() == 1) {
        specialTier.Merge(tiers[0]);
        tiers.erase(tiers.begin());
    }

    tiers.insert(tiers.begin(), specialTier);

    for (ui32 i = 0; i < tiers.size(); ++i) {
        TStringStream ss;
        if (i == 0)
            ss << "SPECIAL:";
        else
            ss << tiers[i].PruneInterval.ToString() << ":";
        for (ui32 src = 0; src < tiers[i].Sources.size(); ++src) {
            ss << tiers[i].Sources[src]->GetDirName() << ",";
        }
        DEBUG_LOG << ss.Str() << Endl;
    }

    return result;
}

void TSearchEnginesManager::Start() {
    IsActive = true;
}

TSearchEnginesManager::TSearchEnginesManager(const TRTYServerConfig& config)
    : IsActive(false)
    , Config(config)
    , EnableMemorySearch(true)
    , PrefetchedVersion(0)
    , SearchVersion(111111111)
    , UnusedVersion(0)
    , ActiveTransactions(0)
    , BaseSearchHandlers(NRTYServer::CreateSearchHandlers(config.GetBaseSearcherConfig(), "Base"))
    , MetaSearchHandlers(NRTYServer::CreateSearchHandlers(config.GetSearcherConfig().ServerOptions, "Meta"))
    , BaseSearchServerMetrics("DiskSearch")
    , MainSearchServerMetrics("MetaSearch")
{
    CommonCache.CreateIfEmpty(Config.GetSearcherConfig().CacheOptions);
    CommonFastCache.CreateIfEmpty(Config.GetSearcherConfig().FastCacheOptions);
    if (!!Config.GetSearcherConfig().EventLog) {
        EventLog = new TEventLog(Config.GetSearcherConfig().EventLog, NEvClass::Factory()->CurrentFormat());
    } else {
        EventLog = new TEventLog(NEvClass::Factory()->CurrentFormat());
    }
    if (Config.GetSearcherConfig().RedirectSearchErrorLogToGlobal) {
        *Singleton<TErrorLog>() = TLoggerOperator<TGlobalLog>::Log();
    }
    if (Config.GetSearcherConfig().WarmupQueries) {
        Prefetcher = MakeHolder<NRTYServer::TPrefetcher>(Config.GetSearcherConfig().WarmupQueries);
    }
    RegisterGlobalMessageProcessor(this);
}

TSearchEnginesManager::~TSearchEnginesManager() {
    UnregisterGlobalMessageProcessor(this);
    VERIFY_WITH_LOG(!IsActive, "TSearchEnginesManager destroying withno Stop before");
    if (!!EventLog)
        EventLog->CloseLog();
}

bool TSearchEnginesManager::IsRemoved(const size_t indexId, const TDocHandle& docId) const {
    if (!docId)
        return false;

    const TSearchersVector& searchers = GetSearchersVectorCurrent();
    if (indexId >= searchers.size())
        return false;

    return searchers[indexId]->IsDeletedDoc(docId);
}

TString TSearchEnginesManager::GetSearchersDataHash(const TSearchersVector& searchersVector) const {
    char buf[25];
    MD5 md5Proc;

    TSearchersVector::const_iterator iter = searchersVector.begin();
    while (iter != searchersVector.end()) {
        TString dataKps = (*iter)->GetSearcherId();
        md5Proc.Update(dataKps.data(), dataKps.size());
        if (Config.GetSearcherConfig().CacheOptions.UseCache) {
            int cache = (*iter)->GetCacheStatus();
            md5Proc.Update(&cache, sizeof(cache));
        }
        if (Config.IsPrefixedIndex) {
            const ui32 shard = (*iter)->GetIndexController()->GetShard();
            md5Proc.Update(&shard, sizeof(shard));
        }
        ++iter;
    }
    if (!Config.GetSearcherConfig().CacheOptions.UseCache) {
        int size = searchersVector.ysize();
        md5Proc.Update(&size, sizeof(size));
    }

    md5Proc.Update(&EnableMemorySearch, sizeof(EnableMemorySearch));

    md5Proc.End_b64(buf);
    return buf;
}

void TSearchEnginesManager::StartTransaction(bool checkMain) {
    TransactionLocker.StartTransaction();
    ActiveTransactions++;
    CHECK_WITH_LOG(!checkMain || ActiveTransactions == 1);
    if (ActiveTransactions == 1) {
        StartConfiguration = GetSearchersDataHash(GetSearchersVectorCurrent());
        INFO_LOG << "Search locked" << Endl;
    }
}

void TSearchEnginesManager::FinishTransaction() {
    if (ActiveTransactions == 1)
        INFO_LOG << "Search unlocking..." << Endl;
    VERIFY_WITH_LOG(ActiveTransactions > 0, "Incorrect usage of FinishTransaction");
    if (ActiveTransactions == 1 && IsActive) {
        if (StartConfiguration != GetSearchersDataHash(GetSearchersVectorNext())) {
            AtomicSet(PrefetchedVersion, SearchVersion + 1);
            StartMetaSearch(GetMetaSearchNext(), GetSearchersVectorNext(), SearchVersion + 1);
            SetActiveSearcherIndex(SearchVersion + 1, GetActiveSearcherIndex(GetSearchersVectorNext()));
            {
                TGuardTransaction gt(SwitchVersionsTransaction);
                INFO_LOG << "Search version incrementation " << SearchVersion << Endl;
                SearchVersion++;
                INFO_LOG << "Search version incremented " << SearchVersion << Endl;
            }
            TGuardTransaction gtMeta(GetMetaLocker(SearchVersion - 1));
            UnusedVersion = SearchVersion - 1;
            TGuardTransaction gtBase(GetBaseLocker(SearchVersion - 1));
            INFO_LOG << "Switch meta start... " << SearchVersion << Endl;
            StopMetaSearch(GetMetaSearchNext());
            GetSearchersVectorNext() = GetSearchersVectorCurrent();
            INFO_LOG << "Switch meta finished " << SearchVersion << Endl;
        }
    }

    ActiveTransactions--;
    bool searchActivated = ActiveTransactions == 0;
    TransactionLocker.FinishTransaction();
    if (searchActivated) {
        TSearchEnginesManager::TGuardedSearchPtr g = GetMetaSearch();
        for (auto&& i : GetSearchersVectorCurrent()) {
            NOTICE_LOG << "Search activated for new source " << i->GetDirName() << " ..." << " / " << i->IsActive() << Endl;
            if (i->IsActive()) {
                i->GetIndexController()->OnSearchActivated();
            }
            NOTICE_LOG << "Search activated for new source " << i->GetDirName() << " done" << Endl;
        }
    }

}

/**
    return dirname of index by index_id as indexes mentioned in "mss"
*/
const TString& TSearchEnginesManager::GetIndexDirName(size_t indexID) const {
    if (indexID < GetSearchersVectorCurrent().size()) {
        return GetSearchersVectorCurrent()[indexID]->GetDirName();
    }
    ythrow yexception() << "Incorrect index_id for determine index directory";
}

TSearchEnginesManager::TGuardedSearchPtr TSearchEnginesManager::GetMetaSearch() const {
    return new TGuardedSearch(*this);
}

TSearchEnginesManager::TGuardedSearchPtr TSearchEnginesManager::GetAppropriateSearcher(const TSearchRequestData& rd) const {
    const TStringBuf path = rd.ScriptName().substr(1);
    if (Y_LIKELY(IsIn(Config.GetSearcherConfig().SearchPath, path))) { // regular search
        if (Config.GetSearcherConfig().DelegationIsActive()) {
            return GetActiveSearch(); // this implements DelegateRequestOptimization
        }

        TSearchEnginesManager::TGuardedSearchPtr meta = GetMetaSearch();
        if (Y_LIKELY(meta && meta->GetMetaSearch() && meta->GetMetaSearch()->IsSearching())) {
            return meta;
        } else {
            return nullptr;
        }
    } else if (IsNumber(path)) { // wormhole to a subsource (eg. /0?text=url:... )
        return GetCommonSearch(FromString<ui64>(path), true);

    } else { // incorrect collection name
        return nullptr;
    }
}

TSearchEnginesManager::TGuardedSearchPtr TSearchEnginesManager::GetActiveSearch() const {
    Y_ASSERT(Config.GetSearcherConfig().DelegationIsActive());
    TSearchEnginesManager::TGuardedSearchPtr g = GetMetaSearch();

    const ui64 active = GetActiveSearcherIndex(g->GetVersion());
    if (Y_UNLIKELY(active >= GetSearchersVectorCurrent().size())) {
        return nullptr;
    }

    Y_ASSERT(GetSearchersVectorCurrent()[active]);
    Y_ASSERT(GetSearchersVectorCurrent()[active]->IsActive());
    return GetCommonSearch(active, true);
}

TSearchEnginesManager::TGuardedSearchPtr TSearchEnginesManager::GetCommonSearch(ui64 index, bool getCurrent) const {
    THolder<TGuardedSearch> guard(new TGuardedSearch(*this, index, getCurrent));
    if (!!guard->GetIndexController()) {
        return guard.Release();
    }
    return nullptr;
}

void TSearchEnginesManager::PrefetchAtStart() const {
    TSearchEnginesManager::TGuardedSearchPtr meta = GetMetaSearch();
    if (meta && meta->GetMetaSearch() && meta->GetMetaSearch()->IsSearching()) {
        try {
            NRTYServer::PrefetchCommon(Prefetcher.Get(), meta->GetMetaSearch());
        } catch (const yexception&) {
            ERROR_LOG << "On meta search prefetch: " << CurrentExceptionMessage() << Endl;
        }
    }
}

ui32 TSearchEnginesManager::GetActiveSearchersCount() const {
    ui32 result = 0;
    for (const auto& i : Searchers) {
        if (i.second->IsActive()) {
            result++;
        }
    }
    return result;
}

ui64 TSearchEnginesManager::GetActiveSearcherIndex(TSearchersVectorConstRef searchers) const {
    for (ui64 i = 0; i < searchers.size(); ++i) {
        auto searcher = searchers[i];
        if (searcher->IsActive()) {
            return i;
        }
    }

    WARNING_LOG << "No active searchers found" << Endl;
    return 0;
}

void TSearchEnginesManager::Stop() {
    TRY
        TGuardTransaction gtCur(ConnectionsLocker[0]);
        TGuardTransaction gtNext(ConnectionsLocker[1]);
        IsActive = false;
        StopMetaSearch(MetaSearch[0]);
        StopMetaSearch(MetaSearch[1]);
    CATCH("while stopping TSearchEnginesManager")
}

void TSearchEnginesManager::StopMetaSearch(TCommonSearchRef metaSearch) {
    if (!!metaSearch) {
        NOTICE_LOG << "Stopping metasearch" << Endl;
        metaSearch->SearchClose();
    }
}

void TSearchEnginesManager::AddSearcher(const TString& dir, IIndexController::TPtr indexController) {
    TGuard<TMutex> g(MutexSearchers);
    TString dirName(TFsPath(dir).Basename());
    DEBUG_LOG << "Add searcher " << SearchVersion + 1 << ", dir =" << dirName << "/ Ptr: " << (ui64)indexController.Get() << Endl;
    VERIFY_WITH_LOG(ActiveTransactions, "Incorrect usage of OnMemoryIndexerStarted without transaction");
    VERIFY_WITH_LOG(Searchers.find(dirName) == Searchers.end(), "Re-add of searcher in SEM: %s", dirName.data());
    VERIFY_WITH_LOG(!Config.GetSearcherConfig().DelegateRequestOptimization || GetActiveSearchersCount() <= 1, "Should be only one searcher when delegate is turning on");
    Searchers[dirName] = new TRTYSearchSource(dirName, indexController, Config);
    if (Searchers[dirName]->IsMem())
        GetSearchersVectorNext().insert(GetSearchersVectorNext().begin(), Searchers[dirName]);
    else
        GetSearchersVectorNext().push_back(Searchers[dirName]);
    NOTICE_LOG << "Added searcher " << SearchVersion + 1 << ", dir = " << dirName << Endl;
}

void TSearchEnginesManager::RemoveSearcher(const TString& dir) {
    TGuard<TMutex> g(MutexSearchers);
    TString dirName(TFsPath(dir).Basename());
    DEBUG_LOG << "Remove searcher " << SearchVersion + 1 << ", dir = " << dirName << Endl;
    VERIFY_WITH_LOG(ActiveTransactions, "Incorrect usage of OnMemoryIndexerStarted without transaction");
    TSearchers::iterator i1 = Searchers.find(dirName);
    VERIFY_WITH_LOG(i1 != Searchers.end(), "Searcher wasn't add to SEM: %s", dirName.data());
    DEBUG_LOG << "Remove searcher " << SearchVersion + 1 << ", dir = " << dirName << " ptr: " << (ui64)i1->second->GetIndexController().Get() << Endl;
    Searchers.erase(dirName);
    for (size_t i2 = 0; i2 < GetSearchersVectorNext().size(); i2++) {
        if (GetSearchersVectorNext()[i2]->GetDirName() == dirName) {
            DEBUG_LOG << "Remove searcher " << SearchVersion + 1 << ", dir = " << dirName << " ptr: " << (ui64)GetSearchersVectorNext()[i2]->GetIndexController().Get() << Endl;
            GetSearchersVectorNext().erase(GetSearchersVectorNext().begin() + i2);
            break;
        }
    }
    DEBUG_LOG << "Removed searcher " << SearchVersion + 1 << ", dir = " << dirName << Endl;
}

bool TSearchEnginesManager::Process(IMessage* message) {
    TMessageRegisterSearcher* messRegisterSearcher = dynamic_cast<TMessageRegisterSearcher*>(message);
    if (messRegisterSearcher) {
        IIndexController::TPtr indexController = messRegisterSearcher->GetIndexController();
        if (indexController->HasSearcher())
            AddSearcher(indexController->Directory().PathName(), indexController);
        return true;
    }

    TMessageUnregisterSearcher* messUnregisterSearcher = dynamic_cast<TMessageUnregisterSearcher*>(message);
    if (messUnregisterSearcher) {
        IIndexController::TPtr indexController = messUnregisterSearcher->GetIndexController();
        if (indexController->HasSearcher())
            RemoveSearcher(indexController->Directory().PathName());
        return true;
    }

    TMessageMemorySearchControl* messMemorySearchControl = dynamic_cast<TMessageMemorySearchControl*>(message);
    if (messMemorySearchControl) {
        TGuardOfTransactionSearchArea gtsa;
        if (messMemorySearchControl->ShouldEnable()) {
            EnableMemorySearch = true;
        } else if (messMemorySearchControl->ShouldDisable()) {
            EnableMemorySearch = false;
        }
        messMemorySearchControl->SetEnabled(EnableMemorySearch);
        return true;
    }

    TMessageCollectServerInfo* messCollect = dynamic_cast<TMessageCollectServerInfo*>(message);
    if (messCollect) {
        messCollect->SetActiveContexts(IReplyContext::GetGlobalActiveObjects());
        messCollect->SetActiveRepliers(ISearchReplier::GetGlobalActiveObjects());
        TMap<TString, bool> copyCacheStates;
        TSearchEnginesManager::TGuardedSearchPtr g = GetMetaSearch();
        ui32 countSearchable = 0;
        for (auto& iter : GetSearchersVectorCurrent()) {
            if (iter->IsActive()) {
                ++countSearchable;
                copyCacheStates[iter->GetDirName()] = iter->GetCacheStatus();
                messCollect->AddSearchableDocs(iter->GetIndexController()->GetSearchableDocumentsCount());
            }
        }
        messCollect->SetCachesStates(copyCacheStates);
        messCollect->SetSearchSourcesCount(countSearchable);
        return true;
    }

    TCollectMetricsMessage* messCollectMetrics = dynamic_cast<TCollectMetricsMessage*>(message);
    if (messCollectMetrics) {
        ui64 diskDocsCount = 0;
        ui64 deletedDocsCount = 0;
        ui64 memoryDocsCount = 0;
        TSearchEnginesManager::TGuardedSearchPtr g = GetMetaSearch();
        for (auto&& source : GetSearchersVectorCurrent()) {
            if (source->IsMem()) {
                memoryDocsCount += source->GetSearchableDocsCount();
            } else if (source->IsActive()) {
                diskDocsCount += source->GetSearchableDocsCount();
                deletedDocsCount += source->GetDeletedDocsCount();
            }
        }
        SearchableIndexMetrics.DiskDocCount.Set(diskDocsCount);
        SearchableIndexMetrics.DiskDeletedDocCount.Set(deletedDocsCount);
        SearchableIndexMetrics.MemoryDocCount.Set(memoryDocsCount);
        return true;
    }

    TMessageGetServerHealth *messHealth = dynamic_cast<TMessageGetServerHealth *>(message);
    if (messHealth) {
        messHealth->ActiveDiskIndexCount = SearchableIndexMetrics.SearchableDiskIndexNumber.Get();
        messHealth->IdleDiskIndexCount = SearchableIndexMetrics.IdleDiskIndexNumber.Get();

        auto drops  = GetMetricResult(MainSearchServerMetrics.DroppedRequests);
        auto totals = GetMetricResult(MainSearchServerMetrics.ResponseTime.TotalMetric());

        // metrics may be empty shortly after the startup
        messHealth->RequestDropRate = drops.Empty() || totals.Empty() || totals->AvgRate < 1e-6
            ? 0. : drops->AvgRate / totals->AvgRate;
        return true;
    }

    TMessageReopenLogs *messReopenLogs = dynamic_cast<TMessageReopenLogs *>(message);
    if (messReopenLogs) {
        TSearchEnginesManager::TGuardedSearchPtr g = GetMetaSearch();
        if (!!g) {
            TRY
                if (!!EventLog) {
                    EventLog->ReopenLog();
                }
                if (g->GetMetaSearch()) {
                    g->GetMetaSearch()->Config->ReopenLog();
                }
            CATCH("while reopenlog");
        }
        return true;
    }

    TMessageGetDocInfo* messDocInfo = dynamic_cast<TMessageGetDocInfo*>(message);
    if (messDocInfo) {
        TSearchEnginesManager::TGuardedSearchPtr g = GetMetaSearch();
        ui32 index;
        ui32 docid;
        if (!DecodeDocId(messDocInfo->DocId, index, docid)) {
            messDocInfo->ErrorMessage = "invalid docid";
            return true;
        }

        TRTYSearchSource& searchSource = *GetSearchersVectorCurrent()[index];
        IIndexController::TPtr indexController = searchSource.GetIndexController();
        VERIFY_WITH_LOG(indexController, "cannot get index controller");

        messDocInfo->HasReply = indexController->GetDocInfo(docid, messDocInfo->Reply);
        messDocInfo->Reply["GLOBAL"].InsertValue("segment", searchSource.GetDirName());

        return true;
    }

    TMessageGetSearchableTimestamp* messSearchableTimestamp = dynamic_cast<TMessageGetSearchableTimestamp*>(message);
    if (messSearchableTimestamp) {
        TSearchEnginesManager::TGuardedSearchPtr g = GetMetaSearch();
        for (auto&& searcher : GetSearchersVectorCurrent()) {
            if (searcher->IsActive()) {
                messSearchableTimestamp->Merge(searcher->GetIndexController()->GetTimestamp());
            }
        }

        return true;
    }

    TMessageUpdateUnistatSignals* messUpdateSignals = dynamic_cast<TMessageUpdateUnistatSignals*>(message);
    if (messUpdateSignals) {
        THashMap<TString, NRTYServer::TBaseTimestamp> realmTimestamps;
        NRTYServer::TBaseTimestamp allTimestamp;
        ui64 diskDocuments = 0;
        ui64 searchableDocuments = 0;
        ui32 state = 0;
        TAreaSearchSourcesInfo mainArea(Config.GetRealmListConfig().GetMainRealmConfig().RealmDirectory, Config.SearchersCountLimit);
        TSearchEnginesManager::TGuardedSearchPtr g = GetMetaSearch();
        for (auto&& source : GetSearchersVectorCurrent()) {
            if (!source->IsMem()) {
                diskDocuments += source->GetSearchableDocsCount();
            }
            if (source->IsActive()) {
                if (mainArea.Contain(*source)) {
                    state = std::max(state, source->GetState());
                }
                const auto& indexTimestamp = source->GetIndexController()->GetTimestamp();

                realmTimestamps[source->GetIndexController()->GetRealmName()].Merge(indexTimestamp);
                allTimestamp.Merge(indexTimestamp);
                searchableDocuments += source->GetSearchableDocsCount();
            }
        }

        TSaasRTYServerSignals::UpdateIndexState(state);
        TSaasRTYServerSignals::UpdateIndexTimestamp("", SafeIntegerCast<ui32>(allTimestamp.GetMax()));
        for (auto&& [realmName, timestamp] : realmTimestamps) {
            TSaasRTYServerSignals::UpdateIndexTimestamp(realmName, SafeIntegerCast<ui32>(timestamp.GetMax()));
        }
        TSaasRTYServerSignals::UpdateIndexDiskDocuments(diskDocuments);
        TSaasRTYServerSignals::UpdateIndexSearchableDocuments(searchableDocuments);
        return true;
    }

    TMessageGetState* messState = dynamic_cast<TMessageGetState*>(message);
    if (messState) {
        TAreaSearchSourcesInfo mainArea(Config.GetRealmListConfig().GetMainRealmConfig().RealmDirectory, Config.SearchersCountLimit);
        TSearchEnginesManager::TGuardedSearchPtr g = GetMetaSearch();
        for (auto&& source : GetSearchersVectorCurrent()) {
            if (source->IsActive() && mainArea.Contain(*source)) {
                messState->State = std::max(messState->State, source->GetState());
            }
        }
        return true;
    }

    TMessageGetIndexHistogram* messHistogram = dynamic_cast<TMessageGetIndexHistogram*>(message);
    if (messHistogram) {
        TSearchEnginesManager::TGuardedSearchPtr g = GetMetaSearch();
        for (auto&& source : GetSearchersVectorCurrent()) {
            if (source->IsActive()) {
                messHistogram->Histograms.Merge(source->GetIndexController()->GetHistograms());
            }
        }
        return true;
    }

    return false;
}

ISearchLocker::ILock* TSearchEnginesManager::GetSearchLock() {
    return new TGuardedSearch(*this);
}
