#include "memory_indexer.h"
#include "memory_indexer_updater.h"

#include "custom_search.h"
#include "memory_search.h"

#include <saas/rtyserver/common/common_messages.h>
#include <saas/rtyserver/common/stream_messages.h>
#include <saas/rtyserver/common/message_collect_server_info.h>
#include <saas/rtyserver/common/search_area_modifier.h>

#include <saas/rtyserver/components/generator/manager.h>
#include <saas/rtyserver/components/generator/parsed_entity.h>
#include <saas/rtyserver/components/special_keys/manager.h>
#include <saas/rtyserver/components/special_keys/const.h>

#include <saas/rtyserver/config/realm_config.h>

#include <saas/rtyserver/search/external_search/rty_index_data.h>
#include <saas/rtyserver/search/config/search_config_builder_memory.h>

#include <saas/rtyserver/indexer_core/guarded_document.h>
#include <saas/rtyserver/indexer_core/index_component_storage.h>

#include <saas/util/logging/exception_process.h>
#include <saas/util/system/copy_recursive.h>

#include <search/base/yx_search.h>
#include <search/request/data/reqdata.h>

namespace {
    void RemoveDirWithContentsIfExists(const TString& dir) {
        if (IsDir(dir))
            RemoveDirWithContents(dir);
    }
}

TMemoryIndexer::TMemoryIndexer(const TString& indexDirName, const TString& tempDirName, const TString& rootDir, const NRTYServer::TIndexerConfig& config, ui32 shard)
    : TIndexerCore(indexDirName, tempDirName, rootDir, config, shard)
    , SearcherConfig(config.Common.Owner.GetSearcherConfig())
    , DocIds(config.MaxDocuments)
    , ConsumeMetrics(GetMetricsPrefix() + "MemoryConsumeTime_Shard_" + ToString(shard))
    , WaitMetrics(GetMetricsPrefix() + "MemoryWaitTime_Shard_" + ToString(shard))
    , Builders(config.Common.Owner, TIndexComponentsStorage::Instance().GetComponentsForOpen())
{
    try {
        TAutoPtr<TSearchConfig> searchConfig(TSearchConfigBuilderMemory(Config.Common.Owner).GetSearchConfig(Dir));

        const NRTYServer::IIndexComponent* comp = TIndexComponentsStorage::Instance().GetComponent(Config.Common.Owner.IndexGenerator);
        CHECK_WITH_LOG(comp);
        TIndexComponentsStorage::Instance().Fill(&Builders, GetBuildersContructionContext());
        CHECK_WITH_LOG(Builders.Start(/* initManagersInterations = */ true));
        Updater.Reset(new TMemoryIndexerUpdater(*this));

        if (comp->IsCommonSearch()) {
            SearchManager.Reset(new TMemorySearchManager(Self, SearcherConfig));
        } else {
            SearchManager.Reset(new TCustomSearchManager(Builders.GetManagers()->GetIndexManager()));
        }
        DDKManager = Builders.GetManagers()->GetDDKManager();
        IndexData.Reset(new TRTYIndexData(Self, &SearcherConfig, SearchManager.Get()));

        if (Builders.GetManagers()->HasIndexManager()) {
            Builders.GetManagers()->GetIndexManager()->LinkIndexData(*this, IndexData.Get());
        }
        if (Config.Common.Owner.GetIndexerMemoryConfig().RealTimeExternalFilesPath.size()) {
            TVector<TFsPath> children;
            TFsPath indexDirectory(searchConfig->IndexDir);
            TFsPath(Config.Common.Owner.GetIndexerMemoryConfig().RealTimeExternalFilesPath).List(children);
            for (auto&& entity: children) {
                if (entity.IsRelative()) {
                    entity = TFsPath::Cwd() / entity;
                }

                const TFsPath& destination = indexDirectory / entity.Basename();
                if (!NFs::SymLink(entity, destination)) {
                    NUtil::CopyRecursive(entity, destination);
                }
            }
        }

        const auto search = SearchManager->GetCommonSearch();
        if (search) {
            comp->TuneSearch(search, *this, IndexData.Get());
        }

        SearchManager->SearchOpen(searchConfig, config, IndexData.Get());
        SearchManager->SignalSearchingStart();

        RegisterIndex();
        RegisterGlobalMessageProcessor(this);
    } catch (...) {
        FAIL_LOG("Can't create memory search: %s", CurrentExceptionMessage().data())
    }
}

TMemoryIndexer::~TMemoryIndexer() noexcept {
    VERIFY_WITH_LOG(Builders.IsClosed(), "Open TMemoryIndexer destroing!!");
}

void TMemoryIndexer::CloseIndex(const std::atomic<bool>* rigidStopSignal, NRTYServer::EExecutionContext) {
    NOTICE_LOG << "status=close_index;dir=" << Directory().BaseName() << Endl;
    UnregisterGlobalMessageProcessor(this);

    TRY
        {
            TGuardOfTransactionSearchArea gtsa;
            UnregisterSearcher();
        }
        TGuardTransaction gtMemoryIndexClosing(TransactionClose);
        if (Builders.IsClosed()) {
            NOTICE_LOG << "status=close_index_duplication;dir=" << Directory().BaseName() << Endl;
            return;
        }

        NOTICE_LOG << "status=unregister_index;dir=" << Directory().BaseName() << Endl;
        SendGlobalMessage<TMessageUnregisterIndex>(this);
        SearchManager->SearchClose();
        if (Builders.GetManagers()->HasIndexManager()) {
            Builders.GetManagers()->GetIndexManager()->UnlinkIndexData();
        }
        NOTICE_LOG << "status=builders_stop;dir=" << Directory().BaseName() << Endl;
        CHECK_WITH_LOG(Builders.Stop());
        NOTICE_LOG << "status=builders_close;dir=" << Directory().BaseName() << Endl;
        CHECK_WITH_LOG(Builders.Close(NRTYServer::TBuilderCloseContext(TempDir, TempDir, nullptr, rigidStopSignal)));
        NOTICE_LOG << "status=remove_directories;dir=" << Directory().BaseName() << ";removed=" << Directory().BaseName() << Endl;
        RemoveDirWithContentsIfExists(Directory().PathName());
        NOTICE_LOG << "status=remove_directories;dir=" << TempDir.BaseName() << ";removed=" << TempDir.BaseName() << Endl;
        RemoveDirWithContentsIfExists(TempDir.PathName());
        return;
    CATCH("While memory indexer " + Directory().BaseName() + " closing")
    FAIL_LOG("Incorrect memory indexer closing");
}

bool TMemoryIndexer::UpdateTimestampsOnDelete(const TQueuedDocument& qDocument, int) {
    VERIFY_WITH_LOG(qDocument->GetCommand() == NRTYServer::TMessage::DELETE_DOCUMENT, "");
    TParsedDocument& document = qDocument->MutableDocument();
    const TDocSearchInfo& docInfo = document.GetDocSearchInfo();

    try {
        UpdateIndexerTimestamp(document);
        return true;
    } catch (...) {
        ERROR_LOG << "while deleting document " << ToString(docInfo.GetKeyPrefix()) << "/" << docInfo.GetUrl() << " to memory indexer " << Directory().BaseName() << " : " << CurrentExceptionMessage() << Endl;
    };
    document.GetErrorsCollector().AddMessage("MEMORY-SEARCHER", "Internal error while deleting document " + ToString(docInfo.GetKeyPrefix()) + " " + docInfo.GetUrl() + " to memory indexer " + Directory().BaseName());
    return false;
}

bool TMemoryIndexer::Index(const TQueuedDocument& qDocument, int /*threadID*/) {
    TParsedDocument& document = qDocument->MutableDocument();
    TBaseGeneratorParsedEntity* indexDoc = document.GetComponentEntity<TBaseGeneratorParsedEntity>(Config.Common.Owner.IndexGenerator);
    const TDocSearchInfo& docInfo = document.GetDocSearchInfo();
    VERIFY_WITH_LOG(indexDoc, "there is no Index component for doc %s", docInfo.GetUrl().data());

    try {
        DEBUG_LOG << "TMemoryIndexer::Index " << TempDir.BaseName() << " start..." << Endl;

        ui32 docId = 0;
        {
            TGuard<TMutex> g(MutexIndex);
            if (Config.Common.Owner.GetIndexerMemoryConfig().DocumentsLimit && DocIds.GetCount() >= Config.Common.Owner.GetIndexerMemoryConfig().DocumentsLimit) {
                DEBUG_LOG << "TMemoryIndexer::Index " << TempDir.BaseName() << " declined by DocumentsLimit" << Endl;
                return true;
            }
            DEBUG_LOG << "TMemoryIndexer::Index " << TempDir.BaseName() << " start...OK" << Endl;

            if (DocIds.GetCount() >= DocIds.Size()) {
                TInstant start = Now();
                for (size_t waitCount = 0; DocIds.GetCount() >= DocIds.Size(); ++waitCount) {
                    if (waitCount % 100 == 0) {
                        DEBUG_LOG << "Memory indexer full waiting (waited " << waitCount << " times)" << Endl;
                    }
                    CVWaitMutex.WaitT(MutexIndex, TDuration::MilliSeconds(100));
                }
                const TDuration waitTime = Now() - start;
                DEBUG_LOG << "Memory indexer full waited " << waitTime.MilliSeconds() << " ms" << Endl;
                WaitMetrics.Hit(waitTime.MilliSeconds());
            }
            DEBUG_LOG << "Indexing document " << docInfo.GetKeyPrefix() << " " << docInfo.GetUrl() << " to memory index " << TempDir.BaseName() << Endl;

            auto indexedDoc = indexDoc->MutableIndexedDoc();

            ui32 docIdRemove = 0;
            const TString& memorySearchUrl = docInfo.GetMSHash();
            const bool keyRemoveNeed = SearchManager->HasDoc(memorySearchUrl, docIdRemove);

            if (SearcherConfig.EnableUrlHash && indexedDoc && !indexedDoc->HasUrlId()) {
                CHECK_WITH_LOG(DDKManager);
                indexedDoc->SetUrlId(DDKManager->GenerateUrlIdHash(docInfo));
            }

            NMemorySearch::EConsumeDataResult dr = SearchManager->Consume(indexedDoc, docInfo, CVWaitMutex, MutexIndex, ConsumeMetrics);
            VERIFY_WITH_LOG(dr == NMemorySearch::CDR_OK, "Incorrect memory indexer consume result %lu %s, %d", docInfo.GetKeyPrefix(), docInfo.GetUrl().data(), (ui32)dr);

            VERIFY_WITH_LOG(SearchManager->HasDoc(memorySearchUrl, docId), "Can't consume document into memory search: %lu %s", docInfo.GetKeyPrefix(), docInfo.GetUrl().data());
            if (keyRemoveNeed && DocIds.HasDoc(docIdRemove)) {
                DEBUG_LOG << "DocId " << docIdRemove << " removed from " << TempDir.BaseName() << Endl;
                DocIds.BanDoc(docIdRemove);
            }
            DEBUG_LOG << "Document has added to memory indexer " << docInfo.GetKeyPrefix() << " " << docInfo.GetUrl() << " " << docId << "/" << Capacity << ". From " << docInfo.GetIndexName() << Endl;
        }
        Builders.Index(0, document, docId);
        {
            TGuard<TMutex> g(MutexIndex);
            DocIds.AddDoc(docId, TRTYMemDocInfo(docInfo.GetIndexName(), docInfo.GetDocId(), docInfo.GetUrl(), docInfo.GetKeyPrefix()));
            UpdateIndexerTimestamp(document);
            TMessageDocumentIndexed messageDocumentIndexed(Directory().PathName(), docInfo.GetUrl(), IndexerType());
            SendGlobalMessage(messageDocumentIndexed);
        }
        return true;
    } catch (...) {
        ERROR_LOG << "while indexing document " << ToString(docInfo.GetKeyPrefix()) << "/" << docInfo.GetUrl() << " to memory indexer " << Directory().BaseName() << " : " << CurrentExceptionMessage() << Endl;
    };
    document.GetErrorsCollector().AddMessage("MEMORY-SEARCHER", "Internal error while indexing document " + ToString(docInfo.GetKeyPrefix()) + " " + docInfo.GetUrl() + " to memory indexer " + Directory().BaseName());
    return false;
}

bool TMemoryIndexer::RemoveDocIdNonLocked(const ui32 docId) {
    const TRTYMemDocInfo& mdi = DocIds.Get(docId);
    if (mdi.IsCorrect) {
        DEBUG_LOG << "action=delete" << ";hash=" << TDocSearchInfo(mdi.Url, mdi.Kps).GetHash().Quote() << ";kps=" << mdi.Kps << ";url=" << mdi.Url << ";docid=" << docId << ";index=" << Directory().BaseName() << ";" << Endl;
        SearchManager->BanDoc(docId);
        DocIds.BanDoc(docId);
        return true;
    } else {
        return false;
    }
}

bool TMemoryIndexer::RemoveUrlsFromTemp(const TString& tempDirName) {
    TInstant removeStart = Now();
    TRY
        DEBUG_LOG << "action=delete_from_temp;temp=" << tempDirName << ";index=" << Directory().BaseName() << ";" << Endl;
        TGuard<TMutex> g(MutexIndex);
        TVector<ui32> docIds;
        docIds.reserve(1024);
        TraverseDocIds([&](ui32 docId, const TRTYMemDocInfo& docInfo) {
            if (docInfo.TempDirName == tempDirName) {
                DEBUG_LOG << "action=delete;kps=" << docInfo.Kps << ";url=" << docInfo.Url << ";docid=" << docId << ";index=" << Directory().BaseName() << ";" << Endl;
                SearchManager->BanDoc(docId);
                DocIds.BanDoc(docId);
                docIds.push_back(docId);
            }
        });
        Builders.GetManagers()->RemoveDocids(docIds);
        CVWaitMutex.Signal();
        DEBUG_LOG << "action=delete_from_temp_finished;temp=" << tempDirName << ";index=" << Directory().BaseName() << ";duration=" << Now() - removeStart << Endl;
        return true;
    CATCH("On remove url set")
    return false;
}

bool TMemoryIndexer::GetDocInfo(const ui32 docId, NJson::TJsonValue& result) const {
    if (docId >= Config.MaxDocuments)
        return false;
    if (!DocIds.HasDoc(docId))
        return false;

    const NRTYServer::IIndexManagersStorage* managers = Builders.GetManagers();
    VERIFY_WITH_LOG(managers, "crash: no managers");
    return managers->GetDocInfo(docId, result);
}

void TMemoryIndexer::FillDocIds(TVector<ui32>& docIds) const {
    docIds.reserve(DocIds.GetCount());
    TraverseDocIds([&docIds](ui32 docId, const TRTYMemDocInfo&) {
        docIds.push_back(docId);
    });
}

ui32 TMemoryIndexer::DoRemoveDocIdsUnsafe(const TVector<ui32>& docIds) {
    ui32 result = 0;
    for (ui32 i = 0; i < docIds.size(); i++)
        if (RemoveDocIdNonLocked(docIds[i]))
            result++;
    Builders.GetManagers()->RemoveDocids(docIds);
    return result;
}

bool TMemoryIndexer::Process(IMessage* message) {
    TMessageRemoveCatchInfo* messRemoveCatch = dynamic_cast<TMessageRemoveCatchInfo*>(message);
    if (messRemoveCatch) {
        TGuardIncompatibleAction giaMemoryIndexClosing(TransactionClose);
        NOTICE_LOG << "RemoveCatchInfo message received for " << TempDir.BaseName() << ", data = " << messRemoveCatch->GetTempName() << Endl;
        if (Builders.IsClosed())
            return true;
        RemoveUrlsFromTemp(messRemoveCatch->GetTempName());
        return true;
    }

    TMessageCollectServerInfo* messCollect = dynamic_cast<TMessageCollectServerInfo*>(message);
    if (messCollect) {
        messCollect->AddDocsInMemoryIndexes(GetDocumentsCount());
        messCollect->AddIndexInfo(*this);
        messCollect->AddMemoryIndexersCount();
        return true;
    }

    TMessageRemoveSpecial* messRemoveSpecial = dynamic_cast<TMessageRemoveSpecial*>(message);
    if (messRemoveSpecial) {
        if (messRemoveSpecial->GetType() == TMessageRemoveSpecial::rtRemoveKps)
            messRemoveSpecial->AddDocuments(RemoveKps(messRemoveSpecial->GetKps()));
        else if (messRemoveSpecial->GetType() == TMessageRemoveSpecial::rtRemoveAll) {
            messRemoveSpecial->AddDocuments(RemoveAll());
        }
        return true;
    }

    TMessageRemoveDeadDocuments* messRemove = dynamic_cast<TMessageRemoveDeadDocuments*>(message);
    if (messRemove) {
        if (!Config.Common.Owner.GetRealmListConfig().GetMainRealmConfig().GetIndexerConfigDisk().TimeToLiveSec) {
            RemoveOldDocuments(messRemove->GetTimeInMinutesGM());
        }
        return true;
    }

    TMessageGetIndexTimestamp* messIndexTimestamp = dynamic_cast<TMessageGetIndexTimestamp*>(message);
    if (messIndexTimestamp) {
        messIndexTimestamp->Merge(*this);
        return true;
    }

    TMessageGetIndexPositions* messIndexPositions = dynamic_cast<TMessageGetIndexPositions*>(message);
    if (messIndexPositions) {
        messIndexPositions->Merge(*this);
        return true;
    }

    return false;
}

bool TMemoryIndexer::DecodeIntoTempDocId(const TDocSearchInfo& info, TDocSearchInfo& result) const {
    TGuard<TMutex> g(MutexIndex);
    const TRTYMemDocInfo& mdi = DocIds.Get(info.GetDocId());
    if (!mdi.IsCorrect) {
        ERROR_LOG << "action=decode_from_mem;docid=" << info.GetDocId() << Endl;
        return false;
    }
    result = TDocSearchInfo(mdi.Url, mdi.TempIndexDocId, mdi.Kps, mdi.TempDirName);
    return true;
}

bool TMemoryIndexer::RemapUrl2DocId(const TDocSearchInfo& docInfo, ui32& docId) const {
    TGuard<TMutex> g(MutexIndex);
    return SearchManager->HasDoc(docInfo.GetMSHash(), docId);
}

bool TMemoryIndexer::RemapUrl2DocIdCandidate(const TDocSearchInfo& docInfo, TDocIdCandidate& docIdCandidate) const {
    TGuard<TMutex> g(MutexIndex);
    ui32 docId;
    bool result = SearchManager->HasDoc(docInfo.GetMSHash(), docId);
    if (result) {
        docIdCandidate.SetDocId(docId);
    }
    return result;
}

IIndexUpdater* TMemoryIndexer::GetUpdater() {
    return Updater.Get();
}

ERequestType TMemoryIndexer::RequestType(const TCgiParameters& cgi) const {
    return SearchManager->RequestType(cgi);
}

TCommonSearch* TMemoryIndexer::GetCommonSearch() const {
    return SearchManager->GetCommonSearch();
}

ui32 TMemoryIndexer::RemoveKps(ui64 kps) {
    TVector<ui32> removingDocIds;
    TraverseDocIds([&removingDocIds, kps](ui32 docId, const TRTYMemDocInfo& docInfo) {
        if (docInfo.Kps == kps) {
            removingDocIds.push_back(docId);
        }
    });
    return RemoveDocIds(removingDocIds);
}

void TMemoryIndexer::TraverseDocIds(std::function<void(ui32, const TRTYMemDocInfo&)>&& operation) const {
    TGuard<TMutex> guard(MutexIndex);
    ui32 docId = 0;
    for (auto i = DocIds.Begin(); i != DocIds.End(); ++i, ++docId) {
        if (i->IsCorrect) {
            operation(docId, *i);
        }
    }
}

ui64 TMemoryIndexer::GetFilesSize() const {
    return 0;
}

ui64 TMemoryIndexer::GetLockedMemorySize() const {
    return 0;
}
