#include "indexer.h"

#include <search/base/yx_search.h>

#include <saas/rtyserver/common/common_messages.h>
#include <saas/rtyserver/common/stream_messages.h>
#include <saas/rtyserver/common/debug_messages.h>
#include <saas/rtyserver/common/fsync.h>
#include <saas/rtyserver/common/search_area_modifier.h>
#include <saas/rtyserver/common/should_stop.h>
#include <saas/rtyserver/components/fullarchive/disk_manager.h>
#include <saas/rtyserver/components/generator/builder.h>
#include <saas/rtyserver/components/special_keys/const.h>
#include <saas/rtyserver/components/special_keys/manager.h>
#include <saas/rtyserver/config/common_indexers_config.h>
#include <saas/rtyserver/config/indexer_config.h>
#include <saas/rtyserver/final_index/final_index_normalizer.h>
#include <saas/rtyserver/indexer_core/file_index_modify_guard.h>
#include <saas/rtyserver/indexer_core/index_component_storage.h>
#include <saas/rtyserver/indexer_core/index_metadata_processor.h>
#include <saas/rtyserver/merger/index_merger.h>
#include <saas/util/logging/exception_process.h>

#include <search/request/data/reqdata.h>
#include <util/system/rwlock.h>

#include <atomic>

class TIndexer::TIndexerCloseCallback: public NRTYServer::IIndexManagersStorage::ICallbacks, public NRTYServer::IIndexBuildersStorage::ICallbacks{
private:
    TIndexer& Indexer;
    THolder<TWriteGuard> WG;
    TRWMutex& Mutex;
public:
    TIndexerCloseCallback(TIndexer& indexer, TRWMutex& mutex)
        : Indexer(indexer)
        , Mutex(mutex)
    {
    }

    void StartGuard() {
        CHECK_WITH_LOG(!WG);
        WG.Reset(new TWriteGuard(Mutex));
        DEBUG_LOG << "On StartGuard for " << Indexer.Directory().PathName() << Endl;
    }

    void StopGuard() {
        CHECK_WITH_LOG(!!WG);
        DEBUG_LOG << "On StopGuard for " << Indexer.Directory().PathName() << Endl;
        WG.Destroy();
    }

    void OnBeforeBuilderClose(const NRTYServer::IIndexComponentBuilder* /*builder*/, const TString& name) override {
        if (name == FULL_ARCHIVE_COMPONENT_NAME) {
            StartGuard();
        }
    }

    void OnAfterBuilderClose(const NRTYServer::IIndexComponentBuilder* /*builder*/, const TString& name) override {
        if (name == FULL_ARCHIVE_COMPONENT_NAME) {
            Indexer.SignalStartRemapUsage();
            StopGuard();
        }
    }

    void OnBeforeManagerClose(const NRTYServer::IIndexComponentManager* manager) override {
        CHECK_WITH_LOG(manager);
        if (manager->GetComponentName() == FULL_ARCHIVE_COMPONENT_NAME) {
            StartGuard();
        }
    }

    void OnAfterManagerClose(const NRTYServer::IIndexComponentManager* manager) override {
        CHECK_WITH_LOG(manager);
        if (manager->GetComponentName() == FULL_ARCHIVE_COMPONENT_NAME) {
            Indexer.BuildReservedFA(Indexer.TempDir.PathName());
            StopGuard();
        }
    }
};

TIndexer::TIndexer(const TString& indexDirName, const TString& tempDirName, const TString& rootDir,
                   const NRTYServer::TIndexerConfig& config, TIndexStorage& indexStorage, unsigned shard)
    : TIndexerCore(indexDirName, tempDirName, rootDir, config, shard)
    , Ttl(config.TimeToLiveSec, config.PhaseToLiveSec)
    , IndexStorage(indexStorage)
    , Builders(config.Common.Owner, TIndexComponentsStorage::Instance().GetComponentsForOpen())
    , IndexedSameUrls(config.AllowSameUrls)
{
    RTY_MEM_LOG("Indexer " + indexDirName + " starting...");
    RestoreIsDeniedFlag = false;
    ManagersClosedFlag = false;
    RemapIsUsefulFlag = false;
    TRY
        TIndexComponentsStorage::Instance().Fill(&Builders, GetBuildersContructionContext());
        Updater.Reset(new TDiskIndexUpdater(*this));
        Builders.Start();
        RegisterIndex();
        RegisterGlobalMessageProcessor(this);

        DDKManager = Builders.GetManagers()->GetDDKManager();
        RTY_MEM_LOG("Indexer " + indexDirName + " start OK");
        return;
    CATCH("while initializing index " + Directory().PathName())
    VERIFY_WITH_LOG(false, "Exception on creating TIndexer! System crashed");
}

TIndexer::~TIndexer() {
    DEBUG_LOG << "Index " << Directory().PathName() << " destroing... " << Endl;
    VERIFY_WITH_LOG(Builders.IsClosed(), "Open indexer destroing: %s", Directory().PathName().data());
}

void TIndexer::KillEmptySegment() {
    Builders.KillBuilders();
    RemoveDirWithContents(TempDir.PathName());
    RemoveDirWithContents(Directory().PathName());
    NOTICE_LOG << "RemoveCatch: " << Directory().PathName() << " ..." << Endl;
    SendGlobalMessage<TMessageRemoveCatchInfo>(Directory().BaseName());
    NOTICE_LOG << "RemoveCatch: " << Directory().PathName() << " done" << Endl;
}

namespace {
    class TFlatDecoder: public TDeferredUpdatesStorage::IDecoder {
    private:
        TVector<ui32> Decoder;
    public:

        TFlatDecoder(const TVector<ui32>& decoder) {
            Decoder = decoder;
        }

        bool Decode(const TDocSearchInfo::THash& /*identifier*/, const ui32 oldDocId, ui32& docid) const override {
            CHECK_WITH_LOG(Decoder.size() > oldDocId);
            docid = Decoder[oldDocId];
            return true;
        }
    };
}

TFinalIndexGuardedPtr TIndexer::BuildIndexFromPortions(TDeferredGuardTransaction& dg, const std::atomic<bool>* rigidStopSignal, NRTYServer::EExecutionContext execContext) {
    TFinalIndexGuardedPtr result;
    NOTICE_LOG << "status=bifp_start;dir=" << Directory().BaseName() << Endl;
    TRY
        TPathName closeIndexDir(TempDir);

        const TBaseGeneratorBuilder* baseIndexGenerator = Builders.GetBuilder<TBaseGeneratorBuilder>(Config.Common.Owner.IndexGenerator);
        VERIFY_WITH_LOG(baseIndexGenerator, "Incorrect index generator: %s", Config.Common.Owner.IndexGenerator.data());
        CHECK_WITH_LOG(!RemapTableOnClose.size());
        ui32 docsCount = baseIndexGenerator->GetResortMapForMerge(this, RemapTableOnClose);
        NOTICE_LOG << "status=bifp;dir=" << Directory().BaseName() << ";resort_docs_count=" << docsCount << Endl;
        if (!docsCount) { // empty segment
            dg.Start();
            KillEmptySegment();
            return TFinalIndexPtr();
        }
        NOTICE_LOG << "status=builders_close_start;dir=" << Directory().BaseName() << Endl;
        {
            TWriteGuard rg(RWFAMutex);
            ManagersClosedFlag = true;
        }
        SendGlobalDebugMessage<TMessageOnCloseIndex, TMessageOnCloseIndex::TStage>(TMessageOnCloseIndex::ManagersClosedFlagSet);

        THashMap<ui32, TString> docIdsHashes;

        const auto hashInit = [this](THashMap<ui32, TString>& hashes) -> bool {
            for (ui32 i = 0; i < RemapTableOnClose.size(); ++i) {
                if (RemapTableOnClose[i] != REMAP_NOWHERE) {
                    hashes[i] = GetDDKManager()->GetIdentifier(i).Quote();
                }
            }
            return true;
        };

        Y_ASSERT(hashInit(docIdsHashes));

        if (IS_LOG_ACTIVE(TLOG_DEBUG) && GetDDKManager()) {
            for (ui32 i = 0; i < RemapTableOnClose.size(); ++i) {
                DEBUG_LOG << "action=decode;hash=" << GetDDKManager()->GetIdentifier(i).Quote() << ";temp_docid=" << i << ";new_docid=" << RemapTableOnClose[i] << ";index=" << Directory().BaseName() << ";removed=" << RemovedDocs.contains(i) << Endl;
            }
        }

        {
            TIndexerCloseCallback callback(*this, RWFAMutex);
            Builders.InitCallbacks(&callback);
            Builders.GetManagers()->InitCallbacks(&callback);
            SendGlobalDebugMessage<TMessageOnCloseIndex, TMessageOnCloseIndex::TStage>(TMessageOnCloseIndex::BeforeBuildersClose);
            CHECK_WITH_LOG(Builders.Close(NRTYServer::TBuilderCloseContext(TempDir, closeIndexDir, &RemapTableOnClose, rigidStopSignal)));
            SendGlobalDebugMessage<TMessageOnCloseIndex, TMessageOnCloseIndex::TStage>(TMessageOnCloseIndex::AfterBuildersClose);
            Builders.GetManagers()->InitCallbacks(nullptr);
            Builders.InitCallbacks(nullptr);
        }

        NOTICE_LOG << "status=builders_close_finish;dir=" << Directory().BaseName() << Endl;
        NOTICE_LOG << Directory().BaseName() << ":" << "DocId2SearchInfo.size() = " << DocId2SearchInfo.size() << Endl;
        if (!DocId2SearchInfo.size()) { // empty segment
            KillEmptySegment();
            return TFinalIndexPtr();
        }
        if (ShouldStop(rigidStopSignal)) {
            return TFinalIndexPtr();
        }
        NOTICE_LOG << "status=check_docids;dir=" << Directory().BaseName() << Endl;
        VERIFY_WITH_LOG(baseIndexGenerator->CheckIndexDocIds(closeIndexDir.PathName()), "Incorrect index was created from portions");
        NOTICE_LOG << "status=normalization;dir=" << Directory().BaseName() << Endl;

        Timestamp.Flush();
        Positions.Flush();

        SendGlobalDebugMessage<TMessageOnCloseIndex, TMessageOnCloseIndex::TStage>(TMessageOnCloseIndex::BeforeBuildServiceFiles);
        if (!TFinalIndexNormalizer::BuildServiceFiles(closeIndexDir, Config.Common.Owner)) {
            ythrow yexception() <<  "while finalize index " << closeIndexDir.PathName();
        }
        SendGlobalDebugMessage<TMessageOnCloseIndex, TMessageOnCloseIndex::TStage>(TMessageOnCloseIndex::AfterBuildServiceFiles);

        FsyncDirRecursively(DirConfig.TempDir);

        NOTICE_LOG << "status=rename;dir=" << Directory().BaseName() << Endl;
        TFsPath resultIndexDir(TFsPath(DirConfig.OldPrefix).Parent());
        resultIndexDir.ForceDelete();
        {
            TWriteGuard rg(RWFAMutex);
            ReservedFAManager.Destroy();
            TFsPath(DirConfig.TempDir).RenameTo(resultIndexDir);
            BuildReservedFA(resultIndexDir);
        }
        SendGlobalDebugMessage<TMessageOnCloseIndex, TMessageOnCloseIndex::TStage>(TMessageOnCloseIndex::AfterRenameDir);
        NOTICE_LOG << "status=rename_ok;dir=" << Directory().BaseName() << Endl;

        SendGlobalDebugMessage<TMessageDeferredUpdaterActivated>(TMessageDeferredUpdaterActivated::EDeferredUpdateMoment::dumCloseIndex);
        result = IndexStorage.AddIndex(Directory().PathName(), TempDir.PathName(), true, &RemapTableOnClose, execContext);
        VERIFY_WITH_LOG(!!result.Get(), "Cannot create final index");
        NOTICE_LOG << "status=frq_restore_start;dir=" << Directory().BaseName() << Endl;
        dg.Start();

        const auto checker = [this, &result, &docIdsHashes]() -> bool {
            for (ui32 idDoc = 0; idDoc < RemapTableOnClose.size(); ++idDoc) {
                if (RemapTableOnClose[idDoc] != REMAP_NOWHERE) {
                    if (TFinalIndexPtr(*result)->GetDDKManager()->GetIdentifier(RemapTableOnClose[idDoc]).Quote() != docIdsHashes[idDoc])
                        return false;
                }
            }
            return true;
        };

        Y_ASSERT(checker());

        {
            TWriteGuard rg(RWFAMutex);
            RestoreIsDeniedFlag = true;
            ReservedFAManager.Destroy();
        }
        TVector<ui32> docidsForDelete;
        TRY // Restore actual indexfrq. Indexation wasn't locked...
            for (ui32 idDoc = 0; idDoc < RemapTableOnClose.size(); ++idDoc) {
                if (IsRemoved(idDoc) && RemapTableOnClose[idDoc] != REMAP_NOWHERE)
                    docidsForDelete.push_back(RemapTableOnClose[idDoc]);
            }
            TFinalIndexPtr(*result)->RemoveDocIds(docidsForDelete);
        CATCH_AND_RETHROW("while sync deleted.log on close index")
        NOTICE_LOG << "status=frq_restore_finish;dir=" << Directory().BaseName() << ";restore_count=" << docidsForDelete.size() << Endl;
        {
            NOTICE_LOG << "status=updates_restore_start;dir=" << Directory().BaseName() << Endl;
            Updater->ApplyUpdatesAndDropStorage(*TFinalIndexPtr(*result)->GetUpdater(), TFlatDecoder(RemapTableOnClose));
            NOTICE_LOG << "status=updates_restore_finish;dir=" << Directory().BaseName() << Endl;
        }
    CATCH_AND_RETHROW("while building index from portions : " + Directory().PathName())
    NOTICE_LOG << "status=bifp_finished;dir=" << Directory().BaseName() << Endl;
    return result;
}

bool TIndexer::UpdateDoc(ui32 docId, const TParsedDocument::TPtr doc) {
    TReadGuard rg(RWFAMutex);
    if (!!ReservedFAManager) {
        return ReservedFAManager->UpdateDoc(docId, doc);
    } else {
        if (ManagersClosedFlag) {
            return true;
        } else {
            CHECK_WITH_LOG(!RestoreIsDeniedFlag);
            auto managers = Builders.GetManagers();
            CHECK_WITH_LOG(managers);
            CHECK_WITH_LOG(managers->IsOpened());
            return managers->UpdateDoc(docId, doc);
        }
    }
}

void TIndexer::BuildReservedFA(const TString& path) {
    if (!!ReservedFAManager)
        ReservedFAManager->Close();
    if (Config.Common.UseSlowUpdate) {
        DEBUG_LOG << "Temporary FA manager build for " << path << Endl;
        auto* manager = Builders.GetManagers()->GetManager<TDiskFAManager>(FULL_ARCHIVE_COMPONENT_NAME);
        if (manager)
            ReservedFAManager.Reset(new TDiskFAManager(path, 0, Config.Common.Owner, Max<ui32>(), manager->GetLayers(), false, true, false));
    }
    if (!!ReservedFAManager)
        CHECK_WITH_LOG(ReservedFAManager->Open());
}

void TIndexer::SignalStartRemapUsage() {
    CHECK_WITH_LOG(!RemapIsUsefulFlag);
    RemapIsUsefulFlag = true;
}

bool TIndexer::RestoreDoc(ui32 docId, TParsedDocument::TPtr& doc) {
    TReadGuard rg(RWFAMutex);
    CHECK_WITH_LOG(!RestoreIsDeniedFlag);
    auto* managers = Builders.GetManagers();
    CHECK_WITH_LOG(managers);
    CHECK_WITH_LOG(managers->IsOpened() || !!ReservedFAManager);
    const TDiskFAManager* fullArc = managers->GetManager<TDiskFAManager>(FULL_ARCHIVE_COMPONENT_NAME);
    if (!!ReservedFAManager) {
        CHECK_WITH_LOG(RemapIsUsefulFlag);
        DEBUG_LOG << "Reserved FA manager usage for " << TempDir.PathName() << " for docid=" << docId << "/" << RemapTableOnClose[docId] << " restoring" << Endl;
        fullArc = ReservedFAManager.Get();
    }
    if (RemapIsUsefulFlag) {
        CHECK_WITH_LOG(RemapTableOnClose.size() > docId);
        DEBUG_LOG << "Indexer remap docid in " << TempDir.PathName() << " for docid=" << docId << " to docid=" << RemapTableOnClose[docId] << Endl;
        docId = RemapTableOnClose[docId];
    }
    if (fullArc) {
        TParsedDocument::TPtr oldDoc = fullArc->GetDoc(docId);
        if (!!oldDoc) {
            doc = oldDoc;
            return true;
        }
    }
    return false;
}

void TIndexer::CloseIndex(const std::atomic<bool>* rigidStopSignal, NRTYServer::EExecutionContext execContext) {
    auto constructionLock = TIndexConstructionLocks::Start(Directory().BaseName());
    TDeferredGuardTransaction callbackLock(TFileIndexModifyGuard::Get()[Shard], ToString(Shard) + " shard transaction");
    NOTICE_LOG << "status=indexer_close;rigid=" << ShouldStop(rigidStopSignal) << ";dir=" << Directory().BaseName() << ";docid=" << DocId << ";docs_count=" << DocId2SearchInfo.size() << Endl;
    bool indexCreated = false;
    TFinalIndexGuardedPtr finalIndexGuard;
    TFinalIndexPtr finalIndex;
    TRY
        if (Builders.IsClosed()) {
            NOTICE_LOG << "status=indexer_close_duplication;dir=" << Directory().BaseName() << Endl;
            return;
        }
        NOTICE_LOG << "status=builders_stop_start;dir=" << Directory().BaseName() << Endl;
        CHECK_WITH_LOG(Builders.Stop());
        NOTICE_LOG << "status=builders_stop_finished;dir=" << Directory().BaseName() << Endl;
        finalIndexGuard = BuildIndexFromPortions(callbackLock, rigidStopSignal, execContext);
        finalIndex = *finalIndexGuard;
        indexCreated = true;
    CATCH("while creating index " + Directory().PathName())
    VERIFY_WITH_LOG(indexCreated, "Index %s wasn't created", Directory().PathName().data());
    NOTICE_LOG << "status=indexer_formed;dir=" << Directory().BaseName() << Endl;
    UnregisterGlobalMessageProcessor(this);
    TRY
        THolder<TGuardOfTransactionSearchArea> guard;
        if ((!!finalIndex && finalIndex->HasSearcher()) || HasSearcher())
            guard.Reset(new TGuardOfTransactionSearchArea());
        UnRegisterIndex();
        if (!!finalIndex)
            finalIndex.Get()->Start();
    CATCH_AND_RETHROW("while closing index " + Directory().PathName())
    NOTICE_LOG << "status=indexer_closed_OK;dir=" << Directory().BaseName() << Endl;
}

bool TIndexer::UpdateTimestampsOnDelete(const TQueuedDocument& qDocument, int) {
    CHECK_WITH_LOG(qDocument->GetCommand() == NRTYServer::TMessage::DELETE_DOCUMENT) << qDocument->GetCommand();
    TParsedDocument& document = qDocument->MutableDocument();
    const TDocSearchInfo& docInfo = document.GetDocSearchInfo();
    TRY
        UpdateIndexerTimestamp(document);
        UpdateIndexerPositions(document);
        return true;
    CATCH("while deleting document " + docInfo.GetUrl());
    document.GetErrorsCollector().AddMessage("DISK-INDEXER", "Internal error while deleting " + docInfo.GetUrl());
    return false;
}

bool TIndexer::Index(const TQueuedDocument& qDocument, int threadID) {
    return Index(qDocument->MutableDocument(), threadID);
}

bool TIndexer::Index(TParsedDocument& document, int threadID) {
    const TDocSearchInfo& docInfo = document.GetDocSearchInfo();
    TRY
        if (Ttl.StartSeconds == 0) {
            Ttl.Start(Now().Seconds(), document.GetVersionTimestamp());
        }
        if (Ttl.IsPhasedMode() && static_cast<TAtomicBase>(document.GetVersionTimestamp()) > Ttl.PhasedStopTimestamp) {
            AtomicSet(Ttl.PhasedStopSignal, 1);
        }

        document.SetTempDiskInfo(AtomicIncrement(DocId) - 1);
        DEBUG_LOG << "action=index_start;url=" << docInfo.GetUrl() << ";kp=" << docInfo.GetKeyPrefix() << ";docid=" << docInfo.GetDocId() << ";hash=" << docInfo.GetHash().Quote() << ";index=" << Directory().BaseName() << Endl;
        if (docInfo.HasIndexName()) {
            TString indexName = docInfo.GetIndexName();
            ui32 hash = FnvHash<ui32>(indexName.data(), indexName.size());
            TWriteGuard g(MutexDocIdsByNextVersionSourceHash);
            DocIdsByNextVersionSourceHash[hash].push_back(docInfo.GetDocId());
        }
        Builders.Index(threadID, document, docInfo.GetDocId());
        document.SetTempDiskInfo(Directory().BaseName());
        UpdateIndexerTimestamp(document);
        UpdateIndexerPositions(document);

        TMessageDocumentIndexed messageDocumentIndexed(Directory().PathName(), docInfo.GetUrl(), IndexerType());
        SendGlobalMessage(messageDocumentIndexed);

        TGuardTransaction giaModifyUrl(TransactionModifyUrl);
        TDocSearchInfo si(docInfo.GetUrl(), docInfo.GetDocId(), docInfo.GetKeyPrefix(), Directory().BaseName());
        const TDocSearchInfo::THash& siHash = si.GetHash();

        THashToDocId::iterator iterUrls = Hash2DocId.find(siHash);
        if (iterUrls != Hash2DocId.end() && !IsRemoved(iterUrls->second)) {
            if(!IndexedSameUrls) {
                FAIL_LOG("Url %s already exists in the current segment", si.GetUrl().c_str());
            }
            RemoveDocIdUnsafe(iterUrls->second);
        }

        Hash2DocId[siHash] = docInfo.GetDocId();
        DocId2SearchInfo.insert(std::make_pair(docInfo.GetDocId(), si)).first->second = si;
        return true;
    CATCH("while indexing document " + docInfo.GetUrl());
    document.GetErrorsCollector().AddMessage("DISK-INDEXER", "Internal error while indexing document " + docInfo.GetUrl());
    return false;
}

ui32 TIndexer::RemoveKps(ui64 kps) {
    TVector<ui32> docIds;
    for (TDocIdToSearchInfo::const_iterator i = DocId2SearchInfo.begin(), e = DocId2SearchInfo.end(); i != e; ++i) {
        const ui32 docid = i->first;
        if (IsRemoved(docid))
            continue;

        if (i->second.GetKeyPrefix() == kps)
            docIds.push_back(docid);
    }
    return RemoveDocIds(docIds);
}

ui32 TIndexer::RemoveAll() {
    TVector<ui32> docIds;
    for (TDocIdToSearchInfo::const_iterator i = DocId2SearchInfo.begin(), e = DocId2SearchInfo.end(); i != e; ++i) {
        const ui32 docid = i->first;
        if (IsRemoved(docid))
            continue;

        docIds.push_back(docid);
    }
    return RemoveDocIds(docIds);
}

bool TIndexer::Process(IMessage* message) {
    TMessageCollectServerInfo* messCollect = dynamic_cast<TMessageCollectServerInfo*>(message);
    if (messCollect) {
        if (!Builders.IsClosed()) {
            messCollect->AddDocsInDiskIndexers(GetDocumentsCount(false));
            messCollect->AddSearchableDocs(GetSearchableDocumentsCount());
        }
        messCollect->AddIndexInfo(*this);
        return true;
    }

    TMessageRemoveCatchInfo* messRemoveCatch = dynamic_cast<TMessageRemoveCatchInfo*>(message);
    if (messRemoveCatch) {
        ui32 hashData = messRemoveCatch->GetHash();
        NOTICE_LOG << "RemoveCatchInfo message received for " << Directory().PathName() << ", data = " << messRemoveCatch->GetTempName() << "/" << hashData << Endl;
        if (!messRemoveCatch->GetExcludedIndexes().contains(Directory().BaseName())) {
            TWriteGuard g(MutexDocIdsByNextVersionSourceHash);
            auto it = DocIdsByNextVersionSourceHash.find(hashData);
            if (it != DocIdsByNextVersionSourceHash.end()) {
                TVector<ui32> docIds = it->second;
                DocIdsByNextVersionSourceHash.erase(it);
                g.Release();
                RemoveDocIds(docIds);
            }
        } else {
            NOTICE_LOG << "Index " << Directory().BaseName() << " excluded for remove cached data = " << messRemoveCatch->GetTempName() << "/" << hashData << Endl;
        }
        return true;
    }

    TMessageRemoveSpecial* messRemoveSpecial = dynamic_cast<TMessageRemoveSpecial*>(message);
    if (messRemoveSpecial) {
        if (messRemoveSpecial->GetType() == TMessageRemoveSpecial::rtRemoveKps)
            messRemoveSpecial->AddNotSearchDocuments(RemoveKps(messRemoveSpecial->GetKps()));
        else if (messRemoveSpecial->GetType() == TMessageRemoveSpecial::rtRemoveAll) {
            messRemoveSpecial->AddNotSearchDocuments(RemoveAll());
        }
        return true;
    }

    TMessageGetIndexTimestamp* messIndexTimestamp = dynamic_cast<TMessageGetIndexTimestamp*>(message);
    if (messIndexTimestamp) {
        messIndexTimestamp->Merge(*this);
        return true;
    }

    TMessageGetIndexPositions* messIndexPositions = dynamic_cast<TMessageGetIndexPositions*>(message);
    if (messIndexPositions) {
        messIndexPositions->Merge(*this);
        return true;
    }

    return false;
}

bool TIndexer::IsRemoved(const ui32 docId) const {
    if (docId > DocId)
        return false;

    TGuardIncompatibleAction giaModifyUrl(TransactionModifyUrl);
    return RemovedDocs.contains(docId);
}

bool TIndexer::RemapUrl2DocIdCandidate(const TDocSearchInfo& docInfo, TDocIdCandidate& docId) const {
    TGuardIncompatibleAction g(TransactionModifyUrl);
    THashToDocId::const_iterator i = Hash2DocId.find(docInfo.GetHash());
    if (i == Hash2DocId.end())
        return false;

    docId.SetDocId(i->second);
    return true;
}


bool TIndexer::RemapUrl2DocId(const TDocSearchInfo& docInfo, ui32& docId) const {
    TGuardIncompatibleAction g(TransactionModifyUrl);
    THashToDocId::const_iterator i = Hash2DocId.find(docInfo.GetHash());
    if (i == Hash2DocId.end())
        return false;

    docId = i->second;
    return true;
}

void TIndexer::StartModification() {
    TransactionModifyUrl.StartTransaction();
}

void TIndexer::FinishModification() {
    TransactionModifyUrl.FinishTransaction();
}

bool TIndexer::RemoveDocIdUnsafe(const ui32 docId) {
    if (docId > DocId)
        return false;

    TDocIdToSearchInfo::iterator iterDocId = DocId2SearchInfo.find(docId);
    VERIFY_WITH_LOG(iterDocId != DocId2SearchInfo.end(), "delete failed %s docid %d", TempDir.BaseName().data(), docId);
    DEBUG_LOG << "action=delete;kp=" << iterDocId->second.GetKeyPrefix() << ";url=" << iterDocId->second.GetUrl() << ";hash=" << iterDocId->second.GetHash().Quote() << ";temp_docid=" << docId << ";index=" << Directory().BaseName() << ";" << Endl;
    VERIFY_WITH_LOG(Hash2DocId.contains(iterDocId->second.GetHash()), "%s: doc %s (%li) is in DocId2SearchInfo, but not in Urls", Directory().PathName().data(), iterDocId->second.GetUrl().data(), iterDocId->second.GetKeyPrefix());
    const bool alreadyRemoved = !RemovedDocs.insert(docId).second;
    if (alreadyRemoved) {
        return false;
    }

    TVector<ui32> docIds(1, docId);
    Updater->RemoveDocs(docIds);
    Builders.GetManagers()->RemoveDocids(docIds);
    return true;
}

ui32 TIndexer::MarkDocIdsForDeleteUnsafe(const TVector<ui32>& docIds, ui32 /*marker*/) {
    return RemoveDocIdsUnsafe(docIds);
}

ui32 TIndexer::GetDocumentsCount(bool keyWithDeleted) const {
    if (keyWithDeleted) {
        return DocId;
    } else {
        TGuardIncompatibleAction gtModifyUrl(TransactionModifyUrl);
        return DocId - RemovedDocs.size();
    }
}

ui32 TIndexer::GetSearchableDocumentsCount() const {
    if (HasSearcher()) {
        return Builders.GetManagers()->GetIndexManager()->GetSearchableDocumentsCount();
    } else {
        return 0;
    }
}

bool TIndexer::IsFull() const {
    if (DocId >= Capacity)
        return true;

    if (!Ttl || DocId == 0)
        return false;

    auto now = Now().Seconds();
    const NRTYServer::TIndexerConfigDisk& config = (Config.RealmConfig ? Config.RealmConfig->GetIndexerConfigDisk() : Config.Common.Owner.GetRealmListConfig().GetMainRealmConfig().GetIndexerConfigDisk());
    if ((DocId > 0) && (Ttl.HadSignal(now) || Ttl.GetDeadline() <= now)) {
        if (!config.WaitCloseForMerger) {
            return true;
        }

        TMessageGetMergerTaskCount messCount(Shard, GetRealmName());
        SendGlobalMessage(messCount);
        if (!messCount.GetCount()) {
            return true;
        }
        Ttl.TimeExtra += config.TimeToLiveExtra;
    }

    return false;
}

ui32 TIndexer::DoRemoveDocIdsUnsafe(const TVector<ui32>& docIds) {
    ui32 result = 0;
    for (int i = 0; i < docIds.ysize(); ++i) {
        if (RemovedDocs.contains(docIds[i]))
            continue;
        if (RemoveDocIdUnsafe(docIds[i]))
            ++result;
    }
    return result;
}

ui64 TIndexer::GetFilesSize() const {
    return 0;
}

ui64 TIndexer::GetLockedMemorySize() const {
    return 0;
}

IIndexUpdater* TIndexer::GetUpdater() {
    return Updater.Get();
}

const IDDKManager* TIndexer::GetDDKManager() const {
    return DDKManager;
}

bool TIndexer::HasSearcher() const {
    return false;
}

TCommonSearch* TIndexer::GetCommonSearch() const {
    return nullptr;
}

TString TIndexer::GetSearcherId() const {
    return "TEMP:" + Directory().BaseName();
}

bool TIndexer::IsSearching() const {
    return Builders.GetManagers()->HasIndexManager() && Builders.GetManagers()->GetIndexManager()->IsSearching();
}

ERequestType TIndexer::RequestType(const TCgiParameters&) const {
    if (IsSearching())
        return RT_Search;
    return RT_Unknown;
}

IIndexController::EIndexType TIndexer::GetType() const {
    return DISK;
}

NRTYServer::ERealm TIndexer::GetRealm() const {
    if (Config.RealmConfig) {
        return Config.RealmConfig->Type;
    } else {
        const TFsPath segment = Directory().PathName();
        return Config.Common.Owner.GetRealmListConfig().GetRealmConfig(segment).Type;
    }
}

TString TIndexer::GetRealmName() const {
    if (Config.RealmConfig) {
        return Config.RealmConfig->ConfigName;
    } else {
        return "Persistent";
    }
}

TString TIndexer::Name() const {
    return TString("Indexer ") + Directory().BaseName();
}
