#include "qs_component.h"
#include "qs_builder.h"
#include "qs_manager.h"
#include "qs_parsed_entity.h"
#include <saas/rtyserver/indexer_core/abstract_model.h>
#include <saas/rtyserver/merger/library/merger.h>
#include <kernel/walrus/yxformbuf.h>
#include <library/cpp/wordpos/wordpos.h>
#include <saas/rtyserver/components/erf/erf_disk.h>
#include <saas/rtyserver/components/erf/erf_writer.h>
#include <saas/rtyserver/components/erf/erf_component.h>
#include <saas/rtyserver/components/fullarchive/disk_manager.h>
#include <saas/rtyserver/config/config.h>
#include <saas/rtyserver/config/searcher_config.h>
#include <saas/rtyserver/config/indexer_config.h>
#include <saas/rtyserver/config/realm_config.h>
#include <saas/rtyserver/factors/factors_config.h>
#include <library/cpp/digest/md5/md5.h>
#include <util/generic/buffer.h>

namespace {
    bool IsUsed(const TRTYServerConfig& Config) {
        return !!Config.GetSearcherConfig().Factors && Config.GetSearcherConfig().Factors->GetQuerySpecFactors().size();
    }
}

TQSIndexComponent::TQSIndexComponent(const TRTYServerConfig& config)
    : IIndexComponent(IsUsed(config))
    , Config(config)
    , FactorsInfo(Config.GetSearcherConfig().Factors->GetQuerySpecFactors())
{
    const bool defaultChecked = false;
    const TIndexFile::EPrefetchPolicy defaultPrefetchPolicy = TIndexFile::ppLock;

    for (TQSFactorsInfo::const_iterator i = FactorsInfo.begin(); i != FactorsInfo.end(); ++i) {
        const TString& erfFile = TNamedBlocksErf::GetErfFileName(i->first);
        IndexFiles.insert(TIndexFile(erfFile, defaultChecked, defaultPrefetchPolicy));
        IndexFiles.insert(TIndexFile(erfFile + ".hdr", defaultChecked, defaultPrefetchPolicy));

        IndexFiles.insert(TIndexFile(TRTYKIReader::GetKeyFilename(i->first), defaultChecked, defaultPrefetchPolicy));
        IndexFiles.insert(TIndexFile(TRTYKIReader::GetInvFilename(i->first), defaultChecked, defaultPrefetchPolicy));
    }
}

NRTYServer::IParsedEntity::TPtr TQSIndexComponent::BuildParsedEntity(NRTYServer::IParsedEntity::TConstructParams& params) const {
    return new TQSParsedEntity(params);
}

NRTYServer::IComponentParser::TPtr TQSIndexComponent::BuildParser() const {
    return new TQSComponentParser{};
}

THolder<NRTYServer::IIndexComponentBuilder> TQSIndexComponent::CreateBuilder(const NRTYServer::TBuilderConstructionContext& context) const {
    if (context.Config.GetType() == "memory")
        return nullptr;
    return MakeHolder<TQSBuilder>(context.TempDir, context.Config, GetName());
}

THolder<NRTYServer::IIndexComponentManager> TQSIndexComponent::CreateManager(const NRTYServer::TManagerConstructionContext& context) const {
    if (context.Config.GetType() == "memory")
        return nullptr;
    return MakeHolder<TQSManager>(context);
}

const TQSIndexComponent::TIndexFiles& TQSIndexComponent::GetIndexFiles() const {
    return IndexFiles;
}

class TQSMergerLogic: public IExternalHitWriter {
private:
    bool H;
    bool L;
    TWordPosition HitLow;
    TWordPosition HitHigh;
    TVector<TRTYErfDiskManagerPtr> InputsErf;
    TVector<TRTYErfDiskManagerPtr> OutputsErf;
    TVector<TErfWriter::TPtr> OutputsErfWriters;
    TBasicFactorStorage FactorsErfBlock;
public:

    TQSMergerLogic(const TRTYMerger::TContext& context, const IRTYStaticFactors* factors, const TString& fileName)
        : FactorsErfBlock(factors->GetStaticFactorsCount())
    {
        H = L = false;
        for (ui32 i = 0; i < context.Sources.size(); ++i) {
            TFsPath path(context.Sources[i]);
            TRTYErfDiskManager::TCreationContext cc(TPathName{path}, fileName, factors, true);
            InputsErf.push_back(new TRTYErfDiskManager(cc, QS_COMPONENT_NAME));
            InputsErf.back()->Open();
        }
        for (ui32 i = 0; i < context.Dests.size(); ++i) {
            TFsPath path(context.Dests[i]);
            TRTYErfDiskManager::TCreationContext cc(TPathName{path}, fileName, factors);
            cc.BlockCount = 1024;
            OutputsErf.push_back(new TRTYErfDiskManager(cc, QS_COMPONENT_NAME));
            OutputsErf.back()->Open();
            OutputsErfWriters.push_back(new TErfWriter(OutputsErf.back().Get()));
        }
    }

    void StartKey(const char* /*key*/) override {
        VERIFY_WITH_LOG(!H && !L, "Incorrect H/L keys");
    }

    void FinishKey() override {
        VERIFY_WITH_LOG(!H && !L, "Incorrect H/L keys");
    }

    void AddHit(ui32 indexSrc, ui32 indexDest, const TSimpleSharedPtr<YxFileWBL>& output, const TWordPosition& hit) override {
        if (hit.GetRelevLevel()) {
            VERIFY_WITH_LOG(hit.GetRelevLevel() == 1, "Incorrect relev key");
            VERIFY_WITH_LOG(!H, "Incorrect relev key - H double");
            H = true;
            HitHigh = hit;
        } else {
            VERIFY_WITH_LOG(!L, "Incorrect relev key - L double");
            L = true;
            HitLow = hit;
        }

        if (H && L) {
            VERIFY_WITH_LOG(HitLow.Doc() == HitHigh.Doc(), "Incorrect docIds for positions");
            VERIFY_WITH_LOG(OutputsErf.size() > indexDest, "Incorrect index dst");
            ui32 posInputErf;
            TQSPosBuilder::BuildPosition(HitLow, HitHigh, posInputErf);
            VERIFY_WITH_LOG(InputsErf[indexSrc]->ReadRaw(FactorsErfBlock, posInputErf), "Incorrect erf position %d", posInputErf);
            ui32 posOutputErf = OutputsErfWriters[indexDest]->Write(FactorsErfBlock);
            TQSPosBuilder::BuildHits(HitLow, HitHigh, HitLow.Doc(), posOutputErf);
            if (HitLow.SuperLong() < HitHigh.SuperLong()) {
                output->AddHit(HitLow.SuperLong());
                output->AddHit(HitHigh.SuperLong());
            } else {
                output->AddHit(HitHigh.SuperLong());
                output->AddHit(HitLow.SuperLong());
            }
            H = L = false;
        }
    }
};

bool TQSIndexComponent::DoMerge(const NRTYServer::TMergeContext& context) const {
    for (TQSFactorsInfo::const_iterator i = FactorsInfo.begin(), e = FactorsInfo.end(); i != e; ++i) {
        TRTYMerger::TContext mc(context.Context);
        TQSMergerLogic qsml(mc, i->second->GetFactorsInfo(), "index." + i->first + ".erf");
        mc.ExternalHitWriter = &qsml;
        TRTYMerger rtyMerger(context.RigidStopSignal, TRTYMerger::otKI);
        // Lock indexation on callback after archive merge
        INFO_LOG << "QSDocs count before merge = " << mc.Decoder->GetSize() << " / " << mc.Decoder->GetNewDocsCount() << Endl;

        mc.AdditionalSuffixIndexName = "." + i->first + ".";
        mc.IgnoreNoAttrs = true;
        if (!rtyMerger.MergeIndicies(mc))
            return false;
        INFO_LOG << "QSDocs count after merge = " << mc.Decoder->GetSize() << " / " << mc.Decoder->GetNewDocsCount() << Endl;
    }
    return true;
}

class TRTYQSErfIndexComponent: public TRTYErfIndexComponent {
public:
    TRTYQSErfIndexComponent(const TRTYServerConfig& config, const NRTYFactors::TQSFactorsHashList::value_type& function)
        : TRTYErfIndexComponent(config, /* isUsed = */ true, "index." + function.first + ".erf", *function.second->GetFactorsInfo())
    {}

    TString GetName() const override {
        return "qserf";
    }
};

bool TQSIndexComponent::DoAllRight(const NRTYServer::TNormalizerContext& context) const
{
    const NRTYFactors::TQSFactorsHashList& factorsList = Config.GetSearcherConfig().Factors->GetQuerySpecFactors();
    for (const auto& function : factorsList) {
        TRTYQSErfIndexComponent erf(Config, function);
        if (!erf.AllRight(context))
            return false;
        try {
            TRTYKIReader reader(context.Dir.PathName(), function.first);
            reader.Open();
        } catch (...) {
            ERROR_LOG << "Invalid QS " << function.first << " in " << context.Dir.PathName() << ": " << CurrentExceptionMessage();
            return false;
        }
    }
    return true;
}

bool TQSIndexComponent::FixByFullArchive(const NRTYServer::TNormalizerContext& context) const {
    const TDiskFAManager* fullArc = context.Managers.GetManager<TDiskFAManager>(FULL_ARCHIVE_COMPONENT_NAME);
    if (!fullArc || !fullArc->IsOpened() || !fullArc->GetLayers().contains(NRTYServer::NFullArchive::FullLayer))
        return false;
    INFO_LOG << "Fix QS in " << context.Dir.PathName() << " by fullarc..." << Endl;
    const NRTYServer::TRealmConfig& realmConfig = Config.GetRealmListConfig().GetRealmConfigByConfigName(context.GetRealmName());
    TQSBuilder dest(context.Dir, realmConfig.GetIndexerConfigDisk(), GetName());
    dest.Start();
    for (auto iter = fullArc->CreateIterator(); iter->IsValid(); iter->Next())
        dest.Index(0, *iter->GetDocument(), iter->GetDocId());
    dest.Stop();
    NRTYServer::TBuilderCloseContext closeContext(context.Dir, context.Dir, nullptr, nullptr);
    NRTYServer::IIndexOwner::TGuardIndexModification g(context.Index);
    dest.Close(closeContext);
    INFO_LOG << "Fix QS in " << context.Dir.PathName() << " by fullarc...OK" << Endl;
    return true;
}

void TQSIndexComponent::CheckAndFix(const NRTYServer::TNormalizerContext& context) const
{
    const NRTYFactors::TQSFactorsHashList& factorsList = Config.GetSearcherConfig().Factors->GetQuerySpecFactors();
    for (const auto& function : factorsList) {
        INFO_LOG << "Fix QS " << function.first << " in " << context.Dir.PathName() << "..." << Endl;
        TString filePath = "index." + function.first;
        TString erfPath = filePath + ".erf";
        try {
            TRTYKIReader reader(context.Dir.PathName(), function.first);
            reader.Open();
        } catch (...) {
            INFO_LOG << "Fix QS " << function.first << " in " << context.Dir.PathName() << " keyinv..." << Endl;
            if (FixByFullArchive(context))
                return;
            NRTYServer::IIndexOwner::TGuardIndexModification g(context.Index);
            TFsPath(context.Dir.PathName() + "/" + erfPath).ForceDelete();
            TFsPath(context.Dir.PathName() + "/" + erfPath + ".hdr").ForceDelete();
            TFsPath(context.Dir.PathName() + "/" + filePath + ".key").ForceDelete();
            TFsPath(context.Dir.PathName() + "/" + filePath + ".inv").ForceDelete();
            TQSMaker maker(1, context.Dir, function.second->GetFactorsInfo(), function.first);
            maker.Start();
            maker.Stop();
            const TPathName dirName(context.Dir.PathName());
            NRTYServer::TBuilderCloseContext closeContex(dirName, dirName, nullptr, nullptr);
            maker.Close(closeContex);
            INFO_LOG << "Fix QS " << function.first << " in " << context.Dir.PathName() << " keyinv...OK" << Endl;
        }
        TRTYQSErfIndexComponent erf(Config, function);
        if (!erf.AllRight(context)) {
            INFO_LOG << "Fix QS " << function.first << " in " << context.Dir.PathName() << " erf..." << Endl;
            erf.CheckAndFix(context);
            INFO_LOG << "Fix QS " << function.first << " in " << context.Dir.PathName() << " erf...OK" << Endl;
        }
    }
}

bool TQSIndexComponent::GetInfoChecker(NRTYServer::TInfoChecker& info) const {
    info.Version = 1;
    MD5 hash;
    const NRTYFactors::TQSFactorsHashList& factorsList = Config.GetSearcherConfig().Factors->GetQuerySpecFactors();
    for (const auto& function : factorsList) {
        TString h = function.first + "." + function.second->GetFactorsInfo()->GetHash();
        hash.Update(h.data(), h.size());
    }
    char buf[33];
    info.AdditionalHash = hash.End(buf);
    return true;
}

