#include "index_storage.h"
#include "index_consume_transaction.h"

#include <saas/rtyserver/common/common_messages.h>
#include <saas/rtyserver/common/doc_utils.h>
#include <saas/rtyserver/common/fsync.h>
#include <saas/rtyserver/common/search_area_modifier.h>
#include <saas/rtyserver/common/sharding.h>
#include <saas/rtyserver/components/fullarchive/disk_manager.h>
#include <saas/rtyserver/components/fullarchive/globals.h>
#include <saas/rtyserver/components/generator/component.h>
#include <saas/rtyserver/config/const.h>
#include <saas/rtyserver/config/shards_config.h>
#include <saas/rtyserver/config/common_indexers_config.h>
#include <saas/rtyserver/config/realm_config.h>
#include <saas/rtyserver/config/indexer_config.h>
#include <saas/rtyserver/config/merger_config.h>
#include <saas/rtyserver/config/repair_config.h>
#include <saas/rtyserver/final_index/final_index_normalizer.h>
#include <saas/rtyserver/indexer_core/file_index_modify_guard.h>
#include <saas/rtyserver/indexer_core/index_component_storage.h>
#include <saas/rtyserver/indexer_core/index_dir.h>
#include <saas/util/logging/exception_process.h>
#include <saas/util/queue.h>
#include <library/cpp/balloc/optional/operators.h>

#include <util/folder/filelist.h>
#include <util/thread/pool.h>
#include <util/system/tempfile.h>
#include <util/system/fs.h>

namespace {
    struct TIndexAdder : public IObjectInQueue {
    public:
        TIndexAdder(TIndexStorage& storage, const TString& index)
            : Storage(storage)
            , Index(index)
        {}

        void Process(void* /*ThreadSpecificResource*/) override {
            ThreadDisableBalloc();
            TRY
                DEBUG_LOG << "AddIndex: " << Index << "..." << Endl;
                Result = Storage.AddIndex(Index, "", true, nullptr, NRTYServer::EExecutionContext::ServerInitialization);
                DEBUG_LOG << "AddIndex: " << Index << "... OK" << Endl;
            CATCH("while adding index " + Index);
        }

        void Start() {
            CHECK_WITH_LOG(!!Result);
            Result->Get()->Start();
        }

    private:
        TFinalIndexGuardedPtr Result;
        TIndexStorage& Storage;
        const TString& Index;
    };

    class TDeferredRemoveTaskImpl : public IIndexStorage::TDeferredRemoveTask {
    public:
        struct TIndexDestructionInfo {
            TFinalIndexPtr Index;
            bool HasSearcher = false;

            TIndexDestructionInfo& SetHasSearcher(bool value = true) {
                HasSearcher = value;
                return *this;
            }

            TIndexDestructionInfo& SetIndex(TFinalIndexPtr index) {
                Index = index;
                return *this;
            }
        };

    private:
        TVector<TIndexDestructionInfo> Indexes;
        TVector<TString> IndexDirs;
        bool DoIndexDestruction;
        const bool IgnoreIndexOrigin;
    public:

        TDeferredRemoveTaskImpl(const TVector<TIndexDestructionInfo> &indexes, bool ignoreIndexOrigin)
            : Indexes(indexes)
            , DoIndexDestruction(true)
            , IgnoreIndexOrigin(ignoreIndexOrigin)
        {
        }

        virtual ~TDeferredRemoveTaskImpl() noexcept(false) {
            TRY
                DEBUG_LOG << "~TDeferredRemoveTaskImpl ..." << Endl;
                DestroyIndexes();
                INFO_LOG << "Apply deferred removes" << Endl;
                for (const auto& indexDir : IndexDirs) {
                    try {
                        RemoveDirWithContents(indexDir);
                        INFO_LOG << "Successfully removed segment dir " << indexDir << Endl;
                    } catch (...) {
                        ERROR_LOG << "Failed to remove segment dir " << indexDir << " - " << CurrentExceptionMessage() << Endl;
                    }
                }
                DEBUG_LOG << "~TDeferredRemoveTaskImpl done" << Endl;
            CATCH("~TDeferredRemoveTaskImpl failed")
        }

        void DestroyIndexes() override {
            if (DoIndexDestruction) {
                TRY
                    DEBUG_LOG << "TDeferredRemoveTaskImpl::DestroyIndexes ..." << Endl;
                    IndexDirs.clear();
                    IndexDirs.reserve(Indexes.size());
                    TSet<TString> excludedIndexes;
                    for (size_t i = 0; i < Indexes.size(); ++i) {
                        excludedIndexes.insert(Indexes[i].Index->Directory().BaseName());
                    }
                    for (size_t i = 0; i < Indexes.size(); ++i) {
                        TString indexDir = Indexes[i].Index->Directory().PathName();
                        TString indexDirName = Indexes[i].Index->Directory().BaseName();
                        NOTICE_LOG << "Deferred remove: " << indexDirName << " ..." << Endl;
                        if (!Indexes[i].HasSearcher && (IgnoreIndexOrigin || !!Indexes[i].Index->GetOriginDir())) {
                            TMessageRemoveCatchInfo messRemoveCatch(indexDirName, excludedIndexes);
                            SendGlobalMessage(messRemoveCatch);
                            NOTICE_LOG << "Deferred RemoveCatch: " << indexDirName << " done" << Endl;
                        }
                        Indexes[i].Index = TFinalIndexPtr();
                        NOTICE_LOG << "Deferred destroy index: " << indexDirName << " done" << Endl;
                        IndexDirs.push_back(indexDir);
                        NOTICE_LOG << "Deferred remove: " << indexDirName << " done" << Endl;
                    }
                    DoIndexDestruction = false;
                    DEBUG_LOG << "TDeferredRemoveTaskImpl::DestroyIndexes done" << Endl;
                CATCH("TDeferredRemoveTaskImpl::DestroyIndexes failed")
            }
        }
    };
} // namespace

void TIndexStorage::UnregisterSearchers(const TVector<TString>& sources) {
    TGuardOfTransactionSearchArea gtsa;
    for (ui32 i = 0; i < sources.size(); ++i) {
        TFinalIndexPtr finalIndex = GetFinalIndex(sources[i]);
        VERIFY_WITH_LOG(!!finalIndex, "Incorrect searchers list: %s", sources[i].data());
        finalIndex->DropSearcher();
    }
}

TIndexStorage::~TIndexStorage() {
    UnregisterGlobalMessageProcessor(&TIndexComponentsStorage::Instance());
};

TIndexStorage::TIndexStorage(const TIndexStorageConstructionContext& context)
    : RealmIndexes({})
    , ExceededLimitRealms({})
    , SegmentId(-1)
    , Config(context.Config)
    , Context(context)
{
    NoSearchableGlobal = false;
    if (!Context.GetSpecificPrefix())
        TIndexComponentsStorage::Instance().ResetConfig(Config);
    RegisterGlobalMessageProcessor(&TIndexComponentsStorage::Instance());
}

struct TDiscoveredSegments {
    TSet<TString> Added;
    TSet<TString> Merged;
    TVector<TString> Broken;
    i32 SegmentId = -1;
};

void TraverseIndexDirectory(const TFsPath& rootDir, const TRTYServerConfig& config, TDiscoveredSegments& segments) {
    TFileEntitiesList list(TFileEntitiesList::EM_DIRS_SLINKS);
    if (!rootDir.Exists()) {
        WARNING_LOG << "Directory with indexes not found. New directory created." << Endl;
        rootDir.MkDirs();
    }
    RTY_MEM_LOG("Init indexes data start");
    list.Fill(rootDir);
    TStringBuf dirName;
    while (dirName = list.Next()) {
        INFO_LOG << "IndexStorage: check " << dirName << Endl;
        NRTYServer::TShard shard = 0;
        i32 segmentId = 0;
        bool isValid = NRTYServer::GetShardAndSegmentId(dirName.data(), shard, segmentId);
        TFsPath dir = rootDir / dirName;
        if (NRTYServer::HasIndexNamePrefix(dirName, DIRPREFIX_MEM) || dirName == "merger" || dirName == "deferred_updates" || dirName.EndsWith("_merge")) {
            dir.ForceDelete();
        } else if (NRTYServer::HasIndexNamePrefix(dirName, DIRPREFIX_INDEX) || NRTYServer::HasIndexNamePrefix(dirName, DIRPREFIX_PREP)) {
            if (isValid) {
                if (!config.GetShardsConfig().IsLocalShard(shard))
                    continue;
                const TBaseIndexComponent* comp = dynamic_cast<const TBaseIndexComponent*>(TIndexComponentsStorage::Instance().GetComponent(config.IndexGenerator));
                CHECK_WITH_LOG(comp);
                if (!comp->IsEmptyIndex(dir)) {
                    segments.SegmentId = std::max(segments.SegmentId, segmentId);
                    segments.Added.insert(dir.GetPath());
                    if ((dir / "source_indexes").Exists()) {
                        TUnbufferedFileInput f(dir / "source_indexes");
                        TString sourceDirName;
                        while (f.ReadLine(sourceDirName)) {
                            segments.Merged.insert((rootDir / sourceDirName).c_str());
                        }
                    }
                } else {
                    WARNING_LOG << "Found empty/broken index: " << dirName << ". Removing it" << Endl;
                    if (config.GetCommonIndexers().Enabled) {
                        RemoveDirWithContents(dir.GetPath());
                    }
                }
            } else {
                WARNING_LOG << "Found intermediate index directory: " << dirName << ". Removing it" << Endl;
                if (config.GetCommonIndexers().Enabled) {
                    RemoveDirWithContents(dir.GetPath());
                }
            }
        } else if (NRTYServer::HasIndexNamePrefix(dirName, DIRPREFIX_TEMP)) {
            if (isValid) {
                segments.SegmentId = std::max(segments.SegmentId, segmentId);
                if (TDiskFAManager::ExistsLayer(dir, NRTYServer::NFullArchive::FullLayer) && (dir / "normal_index").Exists() && config.GetRepairConfig().Enabled) {
                    segments.Broken.push_back(dir.GetPath());
                } else {
                    WARNING_LOG << "Found empty/broken temporary directory: " << dirName << ". Removing it" << Endl;
                    if (config.GetCommonIndexers().Enabled) {
                        RemoveDirWithContents(dir.GetPath());
                    }
                }
            } else {
                WARNING_LOG << "Found intermediate temporary directory: " << dirName << ". Removing it" << Endl;
                if (config.GetCommonIndexers().Enabled) {
                    RemoveDirWithContents(dir.GetPath());
                }
            }
        } else {
            WARNING_LOG << "Unknown folder (" << dirName << ") in segments root " << rootDir << Endl;
        }
    }
}

void TIndexStorage::Init() {
    INFO_LOG << "IndexStorage: init " << Context.GetIndexDirectory() << Endl;
    CanIgnoreIndexOrigin = Context.Config.CanIgnoreIndexOriginOnRemoving;
    if (!Context.GetSpecificPrefix())
        TFileIndexModifyGuard::Reset(Config.GetShardsConfig().Number);

    TFsPath rootDir(Context.GetIndexDirectory());
    {
        TVector<TString> brokenTransactionIndicies;
        TVector<TString> metaFiles;
        TIndexConsumeTransaction::GetIncompletedTransactionIndicies(Context.GetIndexDirectory(), brokenTransactionIndicies, metaFiles);
        for (const auto& brokenTransactionIndexDir : brokenTransactionIndicies) {
            INFO_LOG << "Remove broken transaction index dir: " << brokenTransactionIndexDir << Endl;
            try {
                Y_ENSURE(!brokenTransactionIndexDir.empty()); // Make an extra check in order not to delete the whole index directory
                RemoveDirWithContents(rootDir / brokenTransactionIndexDir);
            } catch (...) {
                ERROR_LOG << "Failed to broken transaction index dir " << brokenTransactionIndexDir << " - " << CurrentExceptionMessage() << Endl;
            }
        }

        for (const auto& metaFilePath: metaFiles) {
            Y_VERIFY(NFs::Remove(metaFilePath)); // all meta files must be deleted or we may suddenly loose index
        }
    }

    TDiscoveredSegments segments;
    for (auto&& [realmName, realmConfig] : Config.GetRealmListConfig().RealmsConfig) {
        if (realmConfig.StorageType == NRTYServer::EStorageType::Disk) {
            TraverseIndexDirectory(realmConfig.RealmDirectory, Config, segments);
        }
    }
    SegmentId = segments.SegmentId + 1;

    for (TSet<TString>::iterator itr = segments.Added.begin(); itr != segments.Added.end();) {
        if (segments.Merged.contains(*itr)) {
            WARNING_LOG << "Found merged index directory: " << *itr << ". Removing it" << Endl;
            RemoveDirWithContents(*itr);
            segments.Added.erase(itr++);
        } else {
            ++itr;
        }
    }

    TRY
        TRTYMtpQueue normalizerQueue(TRTYMtpQueue::TOptions().SetThreadName("IdxStorageNorm"));
        const auto& confRepair = Config.GetRepairConfig();
        normalizerQueue.Start(confRepair.NormalizerThreads);
        RTY_MEM_LOG("Normalization start (" + ToString(confRepair.NormalizerThreads) + "/" + ToString(segments.Added.size()) + ")");
        for (auto&& i: segments.Added) {
            normalizerQueue.SafeAdd(TFinalIndexNormalizer::Create(TPathName{i}, Config, NRTYServer::EExecutionContext::Normalize));
        }
        normalizerQueue.Stop();
        RTY_MEM_LOG("Normalization finished (" + ToString(confRepair.NormalizerThreads) + "/" + ToString(segments.Added.size()) + ")");

        RTY_MEM_LOG("FsyncDirRecursively start");
        for (auto&& i: segments.Added) {
            FsyncDirRecursively(i);
        }
        RTY_MEM_LOG("FsyncDirRecursively finished");

        TFsPath(Context.GetIndexDirectory() + "/normalizer").ForceDelete();
        TRTYMtpQueue addQueue(TRTYMtpQueue::TOptions().SetThreadName("IdxStorageAdd"));
        RTY_MEM_LOG("Add indexes start (" + ToString(confRepair.NormalizerThreads) + "/" + ToString(segments.Added.size()) + ")");
        addQueue.Start(confRepair.NormalizerThreads);
        {
            TVector<TAtomicSharedPtr<TIndexAdder>> adders;
            for (auto&& segment : segments.Added) {
                adders.push_back(new TIndexAdder(*this, segment));
                addQueue.SafeAdd(adders.back().Get());
            }
            addQueue.Stop();
            TGuardOfTransactionSearchArea gtsa;
            for (auto&& i : adders) {
                i->Start();
            }

        }
        RTY_MEM_LOG("Add indexes finished (" + ToString(confRepair.NormalizerThreads) + "/" + ToString(segments.Added.size()) + ")");
        for (unsigned i = 0; i < segments.Broken.size(); i++) {
            TMessageSendRepairInfo message(segments.Broken[i]);
            SendGlobalMessage(message);
        }
    CATCH("Error on Init: FireIndexAdded, FireIndexBroken");
}

void TIndexStorage::Stop() {
}

TVector<TString> TIndexStorage::GetFinalIndexes(const TString& path) const {
    TReadGuard g(ReadWriteMutex);
    return GetFinalIndexesUnsafe(path);
}

TVector<TString> TIndexStorage::GetFinalIndexesUnsafe(const TString& path) const {
    TVector<TString> result;
    for (TMap<TString, TFinalIndexPtr>::const_iterator i = FinalIndexes.begin(), e = FinalIndexes.end(); i != e; ++i) {
        if (!path || TFsPath(path).IsContainerOf(TFsPath(i->second->Directory().PathName())))
            result.push_back(i->first);
    }
    return result;
}

TFinalIndexGuardedPtr TIndexStorage::AddIndex(const TString& pindexDir, const TString& originDir, bool deferredStart, TVector<ui32>* decoder, NRTYServer::EExecutionContext execCtx, bool acquireLock) {
    TString indexDirName(TFsPath(pindexDir).Basename());
    VERIFY_WITH_LOG(pindexDir != indexDirName, "IndexDir should be a path");
    RTY_MEM_LOG("AddIndex " + indexDirName + " start");
    TString indexDir(pindexDir.data()); // for TString thread safety
    INFO_LOG << "locked " << indexDirName << Endl;
    if (!NFs::Exists(indexDir)) {
        WARNING_LOG << "Empty index " << indexDir << Endl;
        return TFinalIndexPtr();
    }

    TFinalIndexPtr finalIndex = TFinalIndex::Create(indexDir, Config, originDir, decoder, execCtx);
    {
        if (acquireLock) {
            ReadWriteMutex.AcquireWrite();
        }
        INFO_LOG << "Index " << indexDirName << " inserted" << Endl;
        VERIFY_WITH_LOG(FinalIndexes.insert(std::make_pair(indexDirName, finalIndex)).second, "Index %s already exists in storage", indexDirName.data());

        const NRTYServer::TRealmConfig& realmConfig = Config.GetRealmListConfig().GetRealmConfig(indexDir);
        if (realmConfig.GetIndexerConfigDisk().MaxSegments && realmConfig.GetIndexerConfigDisk().MaxDocuments + finalIndex->GetDocumentsCount(false) <= realmConfig.GetMergerConfig().MaxDocumentsToMerge) {
            RealmIndexes[realmConfig.ConfigName]++;
            if (RealmIndexes[realmConfig.ConfigName] >= realmConfig.GetIndexerConfigDisk().MaxSegments) {
                ExceededLimitRealms.insert(realmConfig.ConfigName);
                INFO_LOG << "Indexation stopped because MaxSegments limit exceeded for " << ExceededLimitRealms.size() << " realms" << Endl;
            }
        }

        if (acquireLock) {
            ReadWriteMutex.ReleaseWrite();
        }
    }
    if (!deferredStart)
        finalIndex->Start();

    RTY_MEM_LOG("AddIndex " + indexDirName + " finished");
    return finalIndex;
}

ui32 TIndexStorage::GetDocsCount(const TString& dirName, bool withDeleted) const {
    TReadGuard g(ReadWriteMutex);
    TFinalIndexPtr finalIndex = GetFinalIndex(dirName);
    return finalIndex ? finalIndex->GetDocumentsCount(withDeleted) : Max<ui32>();
}

TIndexStorage::TDeferredRemoveTaskPtr TIndexStorage::RemoveIndexes(const TVector<TString>& indexDirs, bool ignoreIndexOrigin) {
    TVector<TDeferredRemoveTaskImpl::TIndexDestructionInfo> indexes;
    indexes.resize(indexDirs.size());
    {
        TWriteGuard g(ReadWriteMutex);
        for (TVector<TString>::const_iterator dir = indexDirs.begin(); dir != indexDirs.end(); ++dir) {
            TMap<TString, TFinalIndexPtr>::iterator i = FinalIndexes.find(TFsPath(*dir).Basename());
            VERIFY_WITH_LOG(i != FinalIndexes.end(), "Index %s doesn't exist in storage", dir->data());

            const NRTYServer::TRealmConfig& realmConfig = Config.GetRealmListConfig().GetRealmConfigByConfigName(i->second->GetRealmName());
            if (realmConfig.GetIndexerConfigDisk().MaxSegments) {
                RealmIndexes[realmConfig.ConfigName]--;
                if (RealmIndexes[realmConfig.ConfigName] < realmConfig.GetIndexerConfigDisk().MaxSegments) {
                    ExceededLimitRealms.erase(realmConfig.ConfigName);
                    if (ExceededLimitRealms.size()) {
                        INFO_LOG << "Indexation is still stopped because MaxSegments limit exceeded for " << ExceededLimitRealms.size() << " realms" << Endl;
                    } else {
                        INFO_LOG << "Indexation not blocked by MaxSegments limit any more" << Endl;
                    }
                }
            }

            indexes[dir - indexDirs.begin()].SetIndex(i->second).SetHasSearcher(i->second->HasSearcher());
            FinalIndexes.erase(i);
        }

    }

    size_t indexWithSearcherCount = 0;
    for (TMap<TString, TFinalIndexPtr>::const_iterator i = FinalIndexes.begin(), e = FinalIndexes.end(); i != e; ++i) {
        if (i->second->HasSearcher()) {
            ++indexWithSearcherCount;
        }
    }

    for (auto&& i: indexes) {
        i.Index->DropSearcher();
    }

    return TDeferredRemoveTaskPtr(new TDeferredRemoveTaskImpl(indexes, CanIgnoreIndexOrigin && ignoreIndexOrigin));
}

TFinalIndexPtr TIndexStorage::GetFinalIndex(const TString& dir) const {
    TReadGuard g(ReadWriteMutex);
    return GetFinalIndexUnsafe(dir);
}

TFinalIndexPtr TIndexStorage::GetFinalIndexUnsafe(const TString& dir) const {
    TMap<TString, TFinalIndexPtr>::const_iterator i = FinalIndexes.find(TFsPath(dir).Basename());
    if (i == FinalIndexes.end())
        return nullptr;

    return i->second;
}

TString TIndexStorage::GetFinalIndexDirs(bool fullPath) const {
    TString indexDirs;
    TReadGuard g(ReadWriteMutex);
    for (TMap<TString, TFinalIndexPtr>::const_iterator i = FinalIndexes.begin(), e = FinalIndexes.end(); i != e; ++i) {
        indexDirs += (fullPath ? i->second->Directory().PathName() : i->second->Directory().BaseName()) + " ";
    }
    return indexDirs;
}

void TIndexStorage::AllocateIndex(TString& indexDirName, TString& tempDirName, int shard /*=0*/, bool isDiskIndex /*=true*/, bool isPrepIndex /*=false*/) {
    TWriteGuard g(ReadWriteMutex);
    AllocateIndexUnsafe(indexDirName, tempDirName, shard, isDiskIndex, isPrepIndex);
}

void TIndexStorage::AllocateIndexUnsafe(TString& indexDirName, TString& tempDirName, int shard /*=0*/, bool isDiskIndex /*=true*/, bool isPrepIndex /*=false*/) {
    if (Config.GetShardsConfig().LocalCount() == 0) {
        VERIFY_WITH_LOG(shard == 0, "Allocating incorrect shard");
    }
    const char* suffix = !isDiskIndex ? "memory_" : isPrepIndex ? "prep_" : "";
    indexDirName = Sprintf("%s%sindex_%010i_%010i", Context.GetSpecificPrefix().data(), suffix, shard, SegmentId);
    tempDirName  = Sprintf("%stemp__%010i_%010i", Context.GetSpecificPrefix().data(), shard, SegmentId);
    DEBUG_LOG << "Allocating " << (isDiskIndex ? "" : "memory ") << "index "
        << indexDirName << " " << tempDirName << " shard:" << shard << " segmentid: " << SegmentId << Endl;
    SegmentId++;
}

bool TIndexStorage::PrepareIndexForConsumption(const TString& indexDirName, bool storeInIndexRoot, TString& finalIndex) {
    const TFsPath consumed(indexDirName);

    NRTYServer::TShard shard = 0;
    i32 segment = 0;
    const bool parsed = NRTYServer::GetShardAndSegmentId(consumed, shard, segment);
    if (Config.GetShardsConfig().LocalCount() > 0 && !parsed) {
        ERROR_LOG << "Cannot parse shard from" << indexDirName << Endl;
        return false;
    }
    if (!NFs::Exists(indexDirName)) {
        ERROR_LOG << indexDirName << " does not exist" << Endl;
        return false;
    }

    const bool isPrepIndex = NRTYServer::HasIndexDirPrefix(consumed, DIRPREFIX_PREP);
    if (!isPrepIndex) {
        if (!TFinalIndexNormalizer::BuildServiceFiles(TPathName(indexDirName), Config)) {
            ERROR_LOG << indexDirName << " normalization failed" << Endl;
            return false;
        }

        if (!IsFinalIndex(indexDirName)) {
            ERROR_LOG << indexDirName << " is not final index after normalization" << Endl;
            return false;
        }
    }

    TString tempIndex;
    AllocateIndexUnsafe(finalIndex, tempIndex, shard, /*isDiskIndex=*/true, /*isPrepIndex=*/ isPrepIndex);

    FsyncDirRecursively(consumed);
    if (storeInIndexRoot) {
        const TFsPath root(Config.GetRealmListConfig().GetMainRealmConfig().RealmDirectory);
        consumed.RenameTo(root / finalIndex);
    }
    return true;
}

TIndexControllerPtr TIndexStorage::ConsumeIndex(const TString& indexDirName, NRTYServer::EConsumeMode mode) {
    TString finalIndex;
    if (!PrepareIndexForConsumption(indexDirName, true, finalIndex)) {
        return nullptr;
    }
    TIndexControllerPtr resultIndex = ConsumeAllocatedIndex(finalIndex, mode);
    INFO_LOG << "Consumed " << indexDirName << " as " << finalIndex << Endl;
    return resultIndex;
}

TIndexControllerPtr TIndexStorage::ConsumeAllocatedIndex(const TString& finalIndex, NRTYServer::EConsumeMode mode) {
    const TFsPath root(Config.GetRealmListConfig().GetMainRealmConfig().RealmDirectory);

    TVector<TFinalIndexPtr> indexes;
    TWriteGuard writeGuard(ReadWriteMutex);
    if (mode == NRTYServer::EConsumeMode::Apply) {
        TVector<TString> indexDirs = GetFinalIndexesUnsafe(root);
        for (const auto& indexDir : indexDirs) {
            indexes.emplace_back(GetFinalIndexUnsafe(indexDir));
        }
    }

    auto indexGuard = AddIndex(root / finalIndex, /*originDir=*/finalIndex, /*deferredStart=*/true, nullptr, NRTYServer::EExecutionContext::Sync, /*acquireLock=*/false);
    auto index = indexGuard->Get();

    if (mode == NRTYServer::EConsumeMode::Apply) {
        TVector<ui32> oldDocsInNewIndex;
        const bool instantDelete = index->GetDocumentsCount(false) == 0;
        INFO_LOG << "Instant delete is set to " << instantDelete << Endl;

        auto docIterator = index->GetManagers().GetDocSearchInfoIterator();
        auto ddkManager = index->GetManagers().GetDDKManager();
        TVector<const IDDKManager*> oldDDKManagers(indexes.size());
        for (ui32 indexId = 0; indexId < indexes.size(); indexId++) {
            oldDDKManagers[indexId] = indexes[indexId]->GetManagers().GetDDKManager();
        }

        TVector<TVector<ui32>> oldDocsInOldIndex(indexes.size());
        ui32 maxDocId = index->GetDocumentsCount(true);

        for (ui32 newDocId = 0; newDocId < maxDocId; newDocId++) {
            auto docIdentifier = ddkManager->GetIdentifier(newDocId);
            if (index->IsRemoved(newDocId)) {
                DEBUG_LOG << "deleted document #" << newDocId << Endl;
            }
            for (ui32 indexId = 0; indexId < indexes.size(); indexId++) {
                ui32 oldDocId = Max<ui32>();
                if (indexes[indexId]->RemapIdentifier2DocId(docIdentifier, oldDocId)) {
                    ui32 currentVersion = ddkManager->GetVersion(newDocId);
                    ui32 previousVersion = oldDDKManagers[indexId]->GetVersion(oldDocId);
                    ui32 currentTimestamp = ddkManager->GetTimeLiveStart(newDocId);
                    ui32 previousTimestamp = oldDDKManagers[indexId]->GetTimeLiveStart(oldDocId);
                    DEBUG_LOG << "doc #" << newDocId << "; old version: " << previousVersion << "; new version: " << currentVersion << Endl;
                    DEBUG_LOG << "old timestamp: " << previousTimestamp << "; new timestamp: " << currentTimestamp << Endl;
                    if (IsDocOutdated(currentVersion, previousVersion, currentTimestamp, previousTimestamp, Config.GetCommonIndexers().TimestampControlEnabled)) {
                        oldDocsInNewIndex.push_back(newDocId);
                        DEBUG_LOG << "removing doc #" << newDocId << " from new index" << Endl;
                    } else {
                        oldDocsInOldIndex[indexId].push_back(oldDocId);
                        DEBUG_LOG << "marking for remove doc #" << oldDocId << " from old index #" << indexId << Endl;
                    }
                }
            }
        }
        INFO_LOG << "removing " << oldDocsInNewIndex.size() << " docs from new index ..." << Endl;
        index->RemoveDocIds(oldDocsInNewIndex);
        INFO_LOG << "removing " << oldDocsInNewIndex.size() << " docs from new index DONE" << Endl;

        for (ui32 indexId = 0; indexId < indexes.size(); indexId++) {
            INFO_LOG << "removing " << oldDocsInOldIndex[indexId].size() << " docs from old index #" << indexId << " ..." << Endl;
            indexes[indexId]->MarkForDelete(oldDocsInOldIndex[indexId], FnvHash<ui32>(finalIndex.data(), finalIndex.size()));
            INFO_LOG << "removing " << oldDocsInOldIndex[indexId].size() << " docs from old index DONE" << Endl;
        }
        if (instantDelete) {
            TMessageRemoveCatchInfo messRemoveCatch(finalIndex);
            SendGlobalMessage(messRemoveCatch);
        }
    }

    index->Start();
    return index;
}

THolder<IIndexConsumeTransaction> TIndexStorage::StartIndexConsumeTransaction() {
    return MakeHolder<TIndexConsumeTransaction>(this, Config.GetRealmListConfig().GetMainRealmConfig().RealmDirectory);
}

bool TIndexStorage::IsFinalIndex(const TString& indexDirOrName) const {
    const TBaseIndexComponent* generator = dynamic_cast<const TBaseIndexComponent*>(TIndexComponentsStorage::Instance().GetComponent(Config.IndexGenerator));
    CHECK_WITH_LOG(generator);
    return generator->IsFinalIndex(GetFullPath(indexDirOrName));
}

bool TIndexStorage::IsEmptyIndex(const TString& indexDirOrName) const {
    const TBaseIndexComponent* comp = dynamic_cast<const TBaseIndexComponent*>(TIndexComponentsStorage::Instance().GetComponent(Config.IndexGenerator));
    CHECK_WITH_LOG(comp);
    return comp->IsEmptyIndex(indexDirOrName);
}

bool TIndexStorage::HasIndexFiles(const TString& indexDirOrName) const {
    const TBaseIndexComponent* comp = dynamic_cast<const TBaseIndexComponent*>(TIndexComponentsStorage::Instance().GetComponent(Config.IndexGenerator));
    CHECK_WITH_LOG(comp);
    return comp->HasIndexFiles(indexDirOrName);
}

TString TIndexStorage::GetFullPath(const TString& indexDirOrName, const TString& path) const {
    if (!path) {
        return (indexDirOrName == TFsPath(indexDirOrName).Basename()) ? Context.GetIndexDirectory() + indexDirOrName : indexDirOrName;
    } else {
        return (indexDirOrName == TFsPath(indexDirOrName).Basename()) ? (path + "/" + indexDirOrName) : indexDirOrName;
    }
}

ui64 TIndexStorage::GetFinalIndexesCount() const {
    TReadGuard guard(ReadWriteMutex);
    return FinalIndexes.size();
}

bool TIndexStorage::ExceededNonMergedIndexesCount() const {
    return ExceededLimitRealms.size();
}
