#include "rty_relevance.h"

#include "factors_calcer.h"
#include "rty_index_text_data.h"
#include "text_machine_plugin.h"
#include "text_machine.h"

#include <saas/rtyserver/factors/rank_model.h>
#include <saas/rtyserver/search/gators/zone_gator.h>
#include <saas/rtyserver/search/ann_features/web_adapter.h>
#include <saas/rtyserver/search/cgi/rty_external_cgi.h>
#include <saas/rtyserver/components/zones_makeup/makeup_manager.h>
#include <saas/rtyserver/components/ann/manager.h>
#include <saas/rtyserver/components/ann/config.h>
#include <saas/rtyserver/components/prop/manager.h>

#include <search/relevance/dynamic_custom_features.h>
#include <search/relevance/query_factors/query_factors.h>
#include <search/lingboost/limits/limits_scheme.h>
#include <search/reqparam/search_specific_lazy_fillers/factor_mask.h>

#include <library/cpp/json/json_writer.h>
#include <kernel/factor_storage/factor_storage.h>

class TRTYInfoPrinter : public TBaseDebugHandlerPrinter {
private:
    const TRTYIndexData* IndexData;
    const IRTYCgiReader* Cgi;
public:

    TRTYInfoPrinter(const TRTYIndexData* indexData, const IRTYCgiReader* cgi) {
        IndexData = indexData;
        Cgi = cgi;
    }

    const TFactorStorage* GetFactors() override {
        return nullptr;
    }

    void Print(const TParamToInfoRequestPrinter& params) override {
        if (!CalcFilteringResultsForSingleDoc(params, DebugHandler, false))
            return;

        const SRelevanceFormula* descr = nullptr;

        if (Cgi && Cgi->GetRankModel() && Cgi->GetRankModel()->GetRankModel() && Cgi->GetRankModel()->GetRankModel()->HasPolynom())
            descr = Cgi->GetRankModel()->GetRankModel()->Polynom()->Descr;

        PrintFormulaAndFactors(
            params,
            DebugHandler.FactorsAndFormulaHandler->Relevance,
            DebugHandler.FactorsAndFormulaHandler->Factors,
            nullptr,
            descr,
            nullptr,
            nullptr,
            ForFastRank);
    }
};

IInfoRequestPrinter* TRTYRelevance::CreateInfoRequestPrinter(const TString& type) {
    if ("rf" == type)
        return new TRTYInfoPrinter(IndexData, Cgi);
    else
        return nullptr;
}

class TRTYRelevance::TDocTimeCalcer : public IDocTimeCalcer {
public:
    TDocTimeCalcer(const TRTYRelevance& owner)
        : Owner(owner)
    {}

    time_t GetTime(ui32 docid) override {
        return Owner.DDKManager->GetTimeLiveStart(docid);
    }
private:
    const TRTYRelevance& Owner;
};


TRTYRelevance::TRTYRelevance(const TFactorsMaskCache& factorsMaskCache, const TRTYIndexData* indexData, const IRTYCgiReader& rtyCgi, const TBaseIndexData& baseIndexData, const TRequestParams* rp)
    : FactorsMaskCache(factorsMaskCache)
    , Cgi(&rtyCgi)
    , Makeup(indexData->GetMakeupManager())
    , Factors(indexData->GetFactorsConfig())
    , DDKManager(indexData->GetDDKManager())
    , ZoneGator(nullptr)
    , IndexData(indexData)
    , BaseIndexData(baseIndexData)
    , MemPool(64 * 1024, TMemoryPool::TLinearGrow::Instance())
    , RP(*rp)
{}

TRTYRelevance::~TRTYRelevance() {
}

void TRTYRelevance::FillPerQueryParams(TFillPerQueryParamsContext& ctx) const {
    FillWebPerQueryParams(ctx);

    //FIXME(SAAS-5876): extract this as IRelevance::OnBeforeFirstPass()
    OnBeforeFirstPass();
}

void TRTYRelevance::CreateGators(TCreateGatorsContext& ctx) const {
    if (Factors->GetZoneFactors().ysize()) {
        ZoneGator = new(ctx.Pool)TZoneGator(ctx.WordWeight, ctx.WordWeightCount, Makeup);
        ctx.GatorRegister->RegisterGator(TRelevanceTypeMask::Text, ZoneGator);
    }
}

#define SET_MODEL_PROPS(getModel, propNamePrefix)                                                             \
{                                                                                                             \
    if (const auto& model = Cgi->getModel()) {                                                                \
        props->Set(#propNamePrefix "ModelName", model->GetName());                                            \
        if (model->GetRankModel()->HasPolynom())                                                              \
            props->Set(#propNamePrefix "ModelFormula", Encode(*model->GetRankModel()->Polynom()->Descr));     \
    }                                                                                                         \
}

void TRTYRelevance::FillProperties(TSearcherProps* props) const {
    if (Cgi->AreRtyFactorsRequested()) {
        FillRtyFactorsProperty(props);
    }
    SET_MODEL_PROPS(GetRankModel, Full);
    SET_MODEL_PROPS(GetFastRankModel, Fast);
    SET_MODEL_PROPS(GetFilterModel, Filter);
    SET_MODEL_PROPS(GetFastFilterModel, FastFilter);
}

void TRTYRelevance::FillRtyFactorsProperty(TSearcherProps* props) const {
    TFactorsCalcer* factorsCalcer = GetCurrentCalcer();
    if (factorsCalcer == nullptr) {
        return;
    }

    if (Factors->GetVersion() < 0) {
        return;
    }

    NJson::TJsonValue factorToIdx(NJson::JSON_MAP);
    for (const ui32 usedFactorIdx: factorsCalcer->GetUsedFactors()) {
        TString factorName = Factors->GetFactorName(usedFactorIdx);
        Y_ASSERT(!factorName.empty());
        factorToIdx[factorName] = usedFactorIdx;
    }

    NJson::TJsonValue resultValue(NJson::JSON_MAP);
    resultValue[::IntToString<10>(Factors->GetVersion())] = NJson::WriteJson(&factorToIdx, /*formatOutput*/ false, /*sortkeys*/ false, /*validateUtf8*/ false);

    props->Set("rty_factors", std::move(resultValue));
}

bool TRTYRelevance::GetSwitchType(ESwitchType& switchType) const {
    //Note: pron=no_factorann also disables Quorum Annotations, unless relev=ann_saas is given, or ComponentsConfig.Ann.UseExternalCalcer is true.
    const TAnnIndexManager* annManager = IndexData->GetAnnManager();
    if (annManager && !Cgi->IsQuorumAnnDisabled(RankingOpts.ReplaceAnnCalcer)) {
        switchType = Cgi->IsSoftnessEnabledInTextQuorum() ? SW_FADR_SAAS : SW_FADR;
        return true;
    }
    return false;
}

TFactorsCalcer* TRTYRelevance::GetMainCalcer() const {
    if (!MainFactorsCalcer && Cgi->GetRankModel())
        MainFactorsCalcer.Reset(new TFactorsCalcer(ZoneGator, *Cgi->GetRankModel(), FactorsMaskCache, &RankingPass, IndexData, Cgi, /*fastFeaturesOnly=*/false));
    return MainFactorsCalcer.Get();
}

TFactorsCalcer* TRTYRelevance::GetFastCalcer() const {
    if (!FastFactorsCalcer && Cgi->GetFastRankModel())
        FastFactorsCalcer.Reset(new TFactorsCalcer(ZoneGator, *Cgi->GetFastRankModel(), FactorsMaskCache, &RankingPass, IndexData, Cgi, /*fastFeaturesOnly=*/true));
    return FastFactorsCalcer.Get();
}

TFactorsCalcer* TRTYRelevance::GetCurrentCalcer() const {
    const bool isFastRankFirstPass = (RP.RequestMode & RPF_FASTRANK) && (RP.IsJustFastRank() || !RankingPass.IsSecond);
    if (Y_UNLIKELY(isFastRankFirstPass)) {
        TFactorsCalcer* fastCalcer = GetFastCalcer();
        if (fastCalcer)
            return fastCalcer;
    }
    return GetMainCalcer();
}

const IFactorsInfo* TRTYRelevance::GetFactorsInfo() const {
    return static_cast<const IFactorsInfo*>(RP.IsJustFastRank() ? GetFastCalcer() : GetMainCalcer());
}

const char* TRTYRelevance::GetFormulaName() {
    auto model = Cgi->GetRankModel();
    return model ? model->GetName().data() : nullptr;
}

namespace {
    void CombineBasesearchFactorMasks(TCombinedFactorMask& fastMask, const TFactorsCalcer* mainCalcer) {
        if (!mainCalcer || !mainCalcer->UsesWebProductionCalcer())
            return;
        const TCombinedFactorMask& mainMask = mainCalcer->GetFactorMask();
        if (mainMask.AllFactors) {
            fastMask = TCombinedFactorMask(true);
        } else {
            fastMask.Group |= mainMask.Group;
            fastMask.Factor |= mainMask.Factor;
        }
    }

    // a checker method for an assertion (debug only)
    [[maybe_unused]]
    inline bool DbgIsCombinedMask(const TCombinedFactorMask& mask, const TCombinedFactorMask& mask1, const TCombinedFactorMask& mask2) {
        struct TDbgFactorMaskAccessor : public TFactorGroupMask {
            using TFactorGroupMask::TFactorGroupMask;
            bool IsCombinedFrom(const TFactorGroupMask& mask1, const TFactorGroupMask& mask2) const {
                return (const TFactorGroupMask::TBase&)(*this) == ((const TFactorGroupMask::TBase&)mask1 | (const TFactorGroupMask::TBase&)mask2);
            }
            static const TDbgFactorMaskAccessor& Cast(const TFactorGroupMask* mask) {
                return *reinterpret_cast<const TDbgFactorMaskAccessor*>(mask);
            }
        };

        const bool expectAllFactors = mask1.AllFactors || mask2.AllFactors;
        if (mask.AllFactors)
            return expectAllFactors;
        else
            return !expectAllFactors && TDbgFactorMaskAccessor::Cast(&mask.Group).IsCombinedFrom(mask1.Group, mask2.Group);
    }
}


void TRTYRelevance::PreInitRequest(TRequestParams* rp) {
    if (!!GetFastCalcer() && (rp->RequestMode & RPF_FASTRANK) && (rp->FastRankDocumentCount.IsSet() || rp->IsJustFastRank())) {
        MakeDestructorHoldingPtr(rp->FactorMask, GetFastCalcer()->GetFactorMask());
        RankingPass.RTYFactorMask = GetFastCalcer()->GetRTYFactorMaskPtr();
        rp->KeepAllDocuments = false;
        if (!rp->IsJustFastRank() && !rp->FactorMask->AllFactors) {
            CombineBasesearchFactorMasks(*rp->FactorMask, GetMainCalcer());
        }
    } else if (!!GetMainCalcer()) {
        rp->RequestMode &= ~RPF_FASTRANK;
        MakeDestructorHoldingPtr(rp->FactorMask, GetMainCalcer()->GetFactorMask());
        RankingPass.RTYFactorMask = GetMainCalcer()->GetRTYFactorMaskPtr();
    }

    if (rp->RelevRegion != END_CATEG) {
        const NGroupingAttrs::TDocsAttrs* da = IndexData->GetDocsAttrs();
        const NGroupingAttrs::TMetainfo* geoa = !da ? nullptr : da->Metainfos().Metainfo("geoa");
        RankingOpts.TmRelevCountry = GetCountry(geoa, rp->RelevRegion);
        if (geoa && rp->AllRegions.empty())
            rp->AllRegions = TRTYAllRegions::MakeAllRegions(geoa, rp->RelevRegion);
    }

    const TAnnIndexManager* annManager = IndexData->GetAnnManager();
    if (annManager) {
        const TAnnComponentConfig& config = annManager->GetConfig();
        const TMaybe<bool> cgiImitateQrQuorum = Cgi->ShouldImitateQrQuorum();
        if (cgiImitateQrQuorum.Defined()) {
            rp->DisableSoleQRposFiltering = cgiImitateQrQuorum.GetRef();
        } else {
            rp->DisableSoleQRposFiltering = config.ImitateQrQuorum ? true : rp->DisableSoleQRposFiltering;
        }
        RankingOpts.ReplaceAnnCalcer = config.UseExternalCalcer || Cgi->ShouldUseExternalAnnCalcer();
        if (RankingOpts.ReplaceAnnCalcer) {
            RankingOpts.UseRegionChain = config.UseRegionChain;
            RankingOpts.AnnRelevRegion = rp->RelevRegion;
            RankingOpts.FactorAnnHitsLimit = Cgi->GetFactorAnnHitsLimit();
            if (!RankingOpts.FactorAnnHitsLimit) {
                RankingOpts.FactorAnnHitsLimit = BaseIndexData.GetBaseIndexStaticData().GetOldFactorAnnHitsLimit();
            }
        }
    }

    if (RankingOpts.ReplaceAnnCalcer) {
        rp->CalcFactorAnn = false; // TDocFeatureCalcer::FactorAnnCalcer will not be created
    }

    TComponentSearcher* presearcher = Cgi->GetPreSearcher();
    if (presearcher && presearcher->CanLookup()) {
        rp->FilterByDocIds = presearcher->Lookup();
        rp->AutoAcceptDocsFromFilterByDocIds = false;
    }
}

void TRTYRelevance::OnBeforeFirstPass() const {
    Y_ASSERT(!RankingPass.IsSecond);
    const bool isFirstPassNotJustFastRank = (RP.RequestMode & RPF_FASTRANK) && !RP.IsJustFastRank();

    if (Y_UNLIKELY(isFirstPassNotJustFastRank)) {
        const bool hasFastDynamicFactors = GetFastCalcer() && GetFastCalcer()->UsesWebProductionCalcer();
        const bool hasMainDynamicfactors = GetMainCalcer() && GetMainCalcer()->UsesWebProductionCalcer();

        if (hasFastDynamicFactors && hasMainDynamicfactors) {
            // here we have DefaultFactorMask(RP.FactorMask) inited as FastMask|MainMask in PreInitRequest()
            Y_ASSERT(DbgIsCombinedMask(DefaultFactorMask(Cgi->GetRP().FactorMask), GetFastCalcer()->GetFactorMask(), GetMainCalcer()->GetFactorMask()));

            // now, after all Gators were created, we can narrow the mask back to FastMask,
            // so runtime may disable some of the features on FastRank (SAASSUP-1810)
            MakeDestructorHoldingPtr(Cgi->GetRP().FactorMask, GetFastCalcer()->GetFactorMask());

            //FIXME(SAAS-5876): move the initialization of RankingPass.RTYFactorMask from PreInitRequest to OnBeforeFirstPass/OnBeforeSecondPass
            Y_ASSERT(RankingPass.RTYFactorMask == GetFastCalcer()->GetRTYFactorMaskPtr());
        }
    }

    OnBeforePass();
}

void TRTYRelevance::OnBeforeSecondPass() {
    // Notes:
    // 1. Assigning DefaultFactorMask(RP.FactorMask) here does almost nothing (legacy code), because
    // TWebBasesearchDynamicFeaturesCalcer is already created, and is not going
    // to apply the changes in FactorMask to the collection of Gators.
    // 2. OnBeforeSecondPass() method may be called two or more times per request.
    if (!RankingPass.IsSecond) {
        MakeDestructorHoldingPtr(Cgi->GetRP().FactorMask, GetMainCalcer()->GetFactorMask());
        RankingPass.IsSecond = true;
        RankingPass.RTYFactorMask = GetMainCalcer()->GetRTYFactorMaskPtr();

        OnBeforePass();
    }
}

void TRTYRelevance::OnBeforePass() const {
    // This is called from both OnBeforeFirstPass() and OnBeforeSecondPass()
    if (RankingOpts.ReplaceAnnCalcer) {
        bool inMask = DefaultFactorMask(RP.FactorMask).Group.Annotation() && (!RankingPass.RTYFactorMask || RankingPass.RTYFactorMask->UsesFactorAnnWithoutTm);
        const NIndexAnn::TReaders* factorAnnReaders = IndexData->GetFactorAnnReaders(BaseIndexData);
        if (factorAnnReaders && inMask) {
            RankingPass.FactorAnnCalcer.Reset(new TRTYFactorAnnFeaturesCalcer);
            RankingPass.FactorAnnCalcer->ResetRequest(*factorAnnReaders, *(RP.GetRequestTree()->Root), RP.AllRegions, RankingOpts);
        } else {
            RankingPass.FactorAnnCalcer.Destroy();
        }
    }
    TFactorsCalcer* factorsCalcer = GetCurrentCalcer();
    if (factorsCalcer)
        factorsCalcer->OnBeforePass();
}

bool TRTYRelevance::ExtractSnippetsHits(ui32 /*docId*/, const TSnippetsHitsContext* /*shc*/) {
    return false;
}
   
void TRTYRelevance::CalcFactors(TCalcFactorsContext& ctx) {
    TFactorsCalcer* calcer = ctx.Fast ? GetFastCalcer() : GetMainCalcer();
    if (calcer) {
        calcer->CalcFactors(ctx);
        auto attrsStoragePtr = calcer->GetUserDefinedAttrs();
        if (attrsStoragePtr) {
            for (const auto& [k, v] : *attrsStoragePtr) {
                UserDefinedAttrs[k] = v;
            }
        }
    }
}

namespace {
    Y_FORCE_INLINE bool HasFilterBorder(const IRTYCgiReader* cgi) {
        return cgi->GetFilterModel() && cgi->GetFilterBorder() > 0;
    }

    Y_FORCE_INLINE bool HasFastFilterBorder(const IRTYCgiReader* cgi) {
        return cgi->GetFastFilterModel() && cgi->GetFastFilterBorder() > 0;
    }

    Y_FORCE_INLINE bool IsHeavyRelevBorder(const IRTYCgiReader* cgi) {
        Y_ASSERT(HasFilterBorder(cgi));
        const auto* filterModel = cgi->GetFilterModel();
        const auto* relevModel = cgi->GetRankModel();
        return filterModel == relevModel && filterModel->GetRankModel() && filterModel->GetRankModel()->HasMatrixnet();
    }

    Y_FORCE_INLINE bool IsLightRelevBorder(const IRTYCgiReader* cgi) {
        Y_ASSERT(HasFastFilterBorder(cgi));
        const auto* filterModel = cgi->GetFastFilterModel();
        const auto* relevModel = cgi->GetFastRankModel();
        return filterModel == relevModel && filterModel->GetRankModel() && filterModel->GetRankModel()->HasMatrixnet();
    }

    Y_FORCE_INLINE bool IsCheatDoc(const IRTYCgiReader* cgi, const TSumRelevParams& srParams) {
        return srParams.CheatParams.IsCheatUrl
            || cgi->GetBorderKeepRefineDoc() && srParams.QueryPrior > 0;
    }

    Y_FORCE_INLINE float GetMxValue(TMaybe<size_t> mxnetIndex, const float* factors, const float defaultValue) {
        return mxnetIndex.Defined() ? factors[*mxnetIndex] : defaultValue;
    }
}

bool TRTYRelevance::ShouldRemoveDocument(TCalcFactorsContext& ctx, TSumRelevParams& srParams) {
    if (ctx.Fast)
        return false;
    if (Y_UNLIKELY(IsCheatDoc(Cgi, srParams)))
        return false;

    if (HasFilterBorder(Cgi) && !IsHeavyRelevBorder(Cgi)) {
        //Note: when FilterModel->HasMatrixNet(), MultiCalc() call below results in x5-x10 CPU overhead (SAAS-5537)
        float filterValue;
        TSumRelevParams* params = (Cgi->GetFilterModel() != Cgi->GetRankModel()) ? nullptr: &srParams;
        TMaybe<size_t> poly_index;
        if(Cgi->GetFbMode() == EFilterBorderMode::Default) {
            poly_index = Cgi->GetFilterModel()->GetFilterPolynomIndex();
        }
        Cgi->GetFilterModel()->MultiCalc(&(ctx.Factors->factors), &filterValue, 1, params,
                            Cgi->GetFilterModel()->GetFilterMatrixNetIndex(), poly_index);

        if (Y_UNLIKELY(Cgi->GetFbMode() == EFilterBorderMode::MatrixNetOnly))
            filterValue = GetMxValue(Cgi->GetFilterModel()->GetMatrixNetIndex(), ctx.Factors->factors, filterValue);
        return Cgi->GetFilterBorder() > filterValue;
    } else {
        return false;
    }
}

bool TRTYRelevance::ShouldRemoveDocument(const TWeighedDoc& doc) {
    //TODO(SAASSUP-1547): document (in comments) that TWeighedDoc::DocId is used as a deletion marker.
    //Second usage: https://nda.ya.ru/3VoT4c (some legacy related to PassFiltration method) .
    return doc.DocId == Max<ui32>();
}

namespace {
    template<typename TFunc>
    Y_FORCE_INLINE void RemoveDocumentByBorder(const IRTYCgiReader* cgi, TCalcRelevanceContext& ctx, float filterBorder, TFunc getFilterValue) {
        for (size_t i = 0; i < ctx.Count; ++i) {
            if (Y_UNLIKELY(IsCheatDoc(cgi, ctx.Params[i])))
                continue;
            if (filterBorder > getFilterValue(i)) {
                ctx.Params[i].DocId = Max<ui32>();
            }
        }
    }
}

void TRTYRelevance::CalcRelevance(TCalcRelevanceContext& ctx) {
    TSumRelevParams::TFactorsExtractor factors(ctx.Params, ctx.Count);
    float** const factorsArray = factors.GetFactors();
    float* const resultRelev = ctx.ResultRelev;
    if (ctx.Fast) {
        const auto model = Cgi->GetFastRankModel();
        if (!model) {
            ythrow yexception() << "Incorrect fastrank usage withno model";
        }

        model->MultiCalc(factorsArray, resultRelev, ctx.Count, nullptr, model->GetFastMatrixNetIndex(), model->GetFastPolynomIndex());

        if (HasFastFilterBorder(Cgi)) {
            const EFilterBorderMode fbMode = Cgi->GetFastFbMode();
            float filterBorder = Cgi->GetFastFilterBorder();
            if (IsLightRelevBorder(Cgi)) {
                const TMaybe<size_t> mxnetIndex = model->GetMatrixNetIndex();
                if (Y_LIKELY(fbMode == EFilterBorderMode::Default || !mxnetIndex.Defined())) {
                    if (Cgi->GetFastFilterModel()->GetFastFilterPolynomIndex().Defined()) { // copying to fictive factor iff it's written in config
                        for (size_t i = 0; i < ctx.Count; ++i) {
                            factorsArray[i][*Cgi->GetFastFilterModel()->GetFastFilterPolynomIndex()] = resultRelev[i];
                        }
                    }
                    if (mxnetIndex.Defined() && Cgi->GetFastFilterModel()->GetFastFilterMatrixNetIndex().Defined()) { // copying to fictive factor iff it's written in config
                        for (size_t i = 0; i < ctx.Count; ++i) {
                            factorsArray[i][*Cgi->GetFastFilterModel()->GetFastFilterMatrixNetIndex()] = factorsArray[i][*mxnetIndex];
                        }
                    }
                    RemoveDocumentByBorder(Cgi, ctx, filterBorder, [resultRelev](size_t i) {
                        return resultRelev[i];// EFilterBorderMode::Default + LightRelevBorder case
                    });
                } else {
                    Y_ASSERT(fbMode == EFilterBorderMode::MatrixNetOnly);
                    if (Cgi->GetFastFilterModel()->GetFastFilterMatrixNetIndex().Defined()) {
                        for (size_t i = 0; i < ctx.Count; ++i) {
                            factorsArray[i][*Cgi->GetFastFilterModel()->GetFastFilterMatrixNetIndex()] = factorsArray[i][*mxnetIndex];
                        }
                    }
                    RemoveDocumentByBorder(Cgi, ctx, filterBorder, [factorsArray, j = *mxnetIndex](size_t i) {
                        return factorsArray[i][j]; // EFilterBorderMode::MatrixNetOnly + LightRelevBorder case
                    });
                }
            } else if ((RP.RequestMode & RPF_FASTRANK) && !RankingPass.IsSecond) {
                TVector<float> filterValues(ctx.Count);
                TMaybe<size_t> poly_index = fbMode == EFilterBorderMode::Default ? Cgi->GetFastFilterModel()->GetFastFilterPolynomIndex() : Nothing();
                Cgi->GetFastFilterModel()->MultiCalc(factorsArray, filterValues.data(), ctx.Count, /*srParamsForCache=*/nullptr,
                                                     Cgi->GetFastFilterModel()->GetFastFilterMatrixNetIndex(), poly_index);
                RemoveDocumentByBorder(Cgi, ctx, filterBorder, [&filterValues](size_t i) {
                    return filterValues[i];
                });
            }
        }
    } else {
        if (!RP.DontCalculateMatrixnet || HasFilterBorder(Cgi) && IsHeavyRelevBorder(Cgi)) {
            Cgi->GetRankModel()->MultiCalc(factorsArray, resultRelev, ctx.Count, ctx.Params,
                                           Cgi->GetRankModel()->GetFullMatrixNetIndex(),
                                           Cgi->GetRankModel()->GetFullPolynomIndex());
        } else {
            for (size_t i = 0; i < ctx.Count; ++i) {
                resultRelev[i] = ctx.Params[i].DocId * 1.f / Max<ui32>();
            }
        }

        if (HasFilterBorder(Cgi)) {
            const EFilterBorderMode fbMode = Cgi->GetFbMode();
            const TMaybe<size_t> mxnetIndex = Cgi->GetRankModel()->GetMatrixNetIndex();
            float filterBorder = Cgi->GetFilterBorder();
            if (IsHeavyRelevBorder(Cgi)) {
                if (Y_LIKELY(fbMode == EFilterBorderMode::Default || !mxnetIndex.Defined())) {
                    if (Cgi->GetFilterModel()->GetFilterPolynomIndex().Defined()) { // copying to fictive factor iff it's written in config
                        for (size_t i = 0; i < ctx.Count; ++i) {
                            factorsArray[i][*Cgi->GetFilterModel()->GetFilterPolynomIndex()] = resultRelev[i];
                        }
                    }
                    if (mxnetIndex.Defined() && Cgi->GetFilterModel()->GetFilterMatrixNetIndex().Defined()) { // copying to fictive factor iff it's written in config
                        for (size_t i = 0; i < ctx.Count; ++i) {
                            factorsArray[i][*Cgi->GetFilterModel()->GetFilterMatrixNetIndex()] = factorsArray[i][*mxnetIndex];
                        }
                    }
                    RemoveDocumentByBorder(Cgi, ctx, filterBorder, [resultRelev](size_t i) {
                        return resultRelev[i];// EFilterBorderMode::Default + HeavyRelevBorder case
                    });
                } else {
                    Y_ASSERT(fbMode == EFilterBorderMode::MatrixNetOnly);
                    if (Cgi->GetFilterModel()->GetFilterMatrixNetIndex().Defined()) {
                        for (size_t i = 0; i < ctx.Count; ++i) {
                            factorsArray[i][*Cgi->GetFilterModel()->GetFilterMatrixNetIndex()] = factorsArray[i][*mxnetIndex];
                        }
                    }
                    RemoveDocumentByBorder(Cgi, ctx, filterBorder, [factorsArray, j = *mxnetIndex](size_t i) {
                        return factorsArray[i][j]; // EFilterBorderMode::MatrixNetOnly + HeavyRelevBorder case
                    });
                }
            } else if ((RP.RequestMode & RPF_FASTRANK) && RankingPass.IsSecond) {
                //this branch is needed because ShouldRemoveDocuments(ctx, srParams) is called only on SinglePassRank, but not on fastrank
                TVector<float> filterValues(ctx.Count);
                TMaybe<size_t> poly_index = fbMode == EFilterBorderMode::Default ? Cgi->GetFilterModel()->GetFilterPolynomIndex() : Nothing();
                Cgi->GetFilterModel()->MultiCalc(factorsArray, filterValues.data(), ctx.Count, /*srParamsForCache=*/nullptr,
                                                Cgi->GetFilterModel()->GetFilterMatrixNetIndex(), poly_index);
                RemoveDocumentByBorder(Cgi, ctx, filterBorder, [&filterValues](size_t i) {
                    return filterValues[i];
                });
            }
        }
    }
}

bool TRTYRelevance::UseWebDocQuorum() const {
    return false;
}

bool TRTYRelevance::UsesVirtualFactorDomain() const {
    return true;
}

TVector<const char*> TRTYRelevance::SerializeAttributes(const TWeighedDoc& doc, TArrayRef<const char * const> attrNames, IAttributeWriter& write, const TRequestResults* /*results*/) const {
    // Note: (yrum, 201909): it is not documented why only the Erf-based static factors are handled here.
    // Some legacy, probably. And it helps to represent ui32 values that are greater than 2^23 (such integers are too big to fit into a float)
    TVector<TStringBuf> attrNamesBuf{attrNames.begin(), attrNames.end()};
    SerializeErfAttributes(doc.DocId, attrNamesBuf, write);
    const auto* prop = IndexData->GetPropManager();
    if (prop && prop->GetConfig()->GetEnableGta()) {
        SerializePropAttributes(doc.DocId, attrNamesBuf, write);
    }
    return {attrNames.begin(), attrNames.end()};
}

TVector<const char*> TRTYRelevance::SerializeFirstStageAttributes(const TWeighedDoc& doc, TArrayRef<const char* const> attrNames, IAttributeWriter& write, const TRequestResults* /*result*/) const {
    TVector<TStringBuf> attrNamesBuf{attrNames.begin(), attrNames.end()};
    SerializeFirstStageFactors(doc, attrNamesBuf, write);
    SerializeErfAttributes(doc.DocId, attrNamesBuf, write);
    SerializeFirstStageProperties(doc.DocId, attrNamesBuf, write);
    SerializeUserDefinedAttributes(doc.DocId, attrNamesBuf, write);
    const auto* prop = IndexData->GetPropManager();
    if (prop && prop->GetConfig()->GetEnableFsgta()) {
        SerializePropAttributes(doc.DocId, attrNamesBuf, write);
    }
    return {attrNames.begin(), attrNames.end()};
}

void TRTYRelevance::SerializeUserDefinedAttributes(ui32 docId, TArrayRef<const TStringBuf> names, IAttributeWriter& write) const {
    for (const auto& name: names) {
        if (const auto* value = UserDefinedAttrs.FindPtr(std::pair(docId, name))) {
            write(name, *value);
        }
    }
}

void TRTYRelevance::SerializeErfAttributes(ui32 docId, TArrayRef<const TStringBuf> names, IAttributeWriter& write) const {
    constexpr TStringBuf erfPrefix{"_Erf_"};
    const IRTYErfManager* erf = IndexData->GetErfManager();
    if (!erf) {
        return;
    }

    const IRTYStaticFactors* factors = erf->GetFactorsInfo();
    Y_ASSERT(factors);

    TVector<std::pair<TStringBuf, ui32>> fields(Reserve(names.size()));

    for (const auto& name: names) {
        auto erfName = name;
        if (erfName.SkipPrefix(erfPrefix)) {
            ui32 index = 0;
            if (factors->CheckFactorName(erfName, index)) {
                fields.emplace_back(name, index);
            }
        }
    }

    if (fields.empty()) {
        return;
    }

    TBasicFactorStorage erfBlock(factors->GetStaticFactorsCount());
    if (!erf->ReadRaw(erfBlock, docId)) {
        return;
    }

    for (const auto& [name, index]: fields) {
        const float value = erfBlock[index];
        TString stringValue = factors->GetFactor(index).Type == TBitsReader::ftInt
            ? ToString(ui32(value))
            : ToString(value);
        write(name, stringValue);
    }
}

void TRTYRelevance::SerializePropAttributes(ui32 docId, TArrayRef<const TStringBuf> names, IAttributeWriter& write) const {
    THashSet<TStringBuf> attrSet{names.begin(), names.end()};
    const bool includeAll = attrSet.contains("_AllDocInfos");
    const auto* prop = IndexData->GetPropManager();
    prop->WalkDocumentProperties(docId, [&](TStringBuf name, TStringBuf value) {
        if (includeAll || attrSet.contains(name)) {
            write(name, value);
        }
    });
}

void TRTYRelevance::SerializeFirstStageProperties(ui32 docId, TArrayRef<const TStringBuf> names, IAttributeWriter& write) const {
    constexpr TStringBuf DP_ZONE_TITLE = "_DocZoneTitle";

    if (std::find(names.begin(), names.end(), DP_ZONE_TITLE) == names.end()) {
        return;
    }

    const ISentenceZonesReader* zones = Makeup; // may be nullptr
    const IArchiveData* ad = BaseIndexData.GetArchiveManager();
    if (!ad) {
        return;
    }
    const TBlob docText = ad->GetDocText(docId)->UncompressBlob();
    if (docText.Empty()) {
        return;
    }

    const TRTYIndexTextData& textOpts = IndexData->GetIndexTextOpts();

    const TString& value = NRTYServer::GetArcTitle(docText.AsUnsignedCharPtr(), docId, zones, textOpts.GetTitleZonesMask(), textOpts.GetAutoTitleSentNum());
    write(DP_ZONE_TITLE, value);
}

void TRTYRelevance::SerializeFirstStageFactors(const TWeighedDoc& doc, TArrayRef<const TStringBuf> names, IAttributeWriter& write) const {
    if (!ValidAndHasFactors(doc.FactorStorage))
        return;

    TConstFactorView view = doc.FactorStorage->CreateConstView();

    for (const auto& name: names) {
        // XXX(eeight) name.data() is not guaranteed to be null-terminated.
        if (const auto index = GetFactorsInfo()->GetFactorIndex(name.data())) {
            Y_ASSERT(*index < view.Size());
            write(name, ToString(view[*index]));
        }
    }
}

IDocTimeCalcer* TRTYRelevance::CreateDocTimeCalcer() {
    return new TDocTimeCalcer(*this);
}

ICreatorOfCustomDynamicFeaturesCalcer* TRTYRelevance::CreateCustomDynamicFeaturesCalcerCreator() {
    return new NRTYServer::TCreatorOfDynamicFeatures(&RankingOpts);
}

const TMap<ui32, NTextMachineProtocol::TPbHits>* TRTYRelevance::GetTextMachineDocHits() const {
    return RankingPass.TmCustomCalcer ? &RankingPass.TmCustomCalcer->GetStats().GetDocHits() : nullptr;
}

const TMap<ui32, NTextMachineProtocol::TPbLimitsMonitor>* TRTYRelevance::GetTextMachineDocLimitsMonitors() const {
    return RankingPass.TmCustomCalcer ? &RankingPass.TmCustomCalcer->GetStats().GetDocLimitsMonitors() : nullptr;
}

const THashMap<ui32, NTextMachineProtocol::TPbQueryWordStats>* TRTYRelevance::GetTextMachineDocWordStats() const {
    return RankingPass.TmCustomCalcer ? &RankingPass.TmCustomCalcer->GetStats().GetDocWordStats() : nullptr;
}

TTextMachineFeaturesCalcer* TRTYRelevance::CreateExternalTextMachineCalcer(const bool needOpenIterators) {
    TSaasTextMachineCalcerHolder holder = DoCreateExternalTextMachineCalcer(needOpenIterators);

    if (holder.IsCustom()) {
        RankingPass.TmCustomCalcer = std::move(holder.CustomCalcer);

        TFactorsCalcer* factorsCalcer = GetCurrentCalcer();
        if (factorsCalcer)
            factorsCalcer->OnReopenTextMachine();
        return nullptr;
    } else {
        RankingPass.TmCustomCalcer.Drop();
        return holder.WebProductionCalcer.Release();
    }
}

TSaasTextMachineCalcerHolder TRTYRelevance::DoCreateExternalTextMachineCalcer(const bool needOpenIterators) {
    if (!RP.RequestBundle) {
        return {};
    }
    if ((RP.RequestMode & RPF_FASTRANK) && !RankingPass.IsSecond && !RP.UseTextMachineOnL2.Get()) {
        return {};
    }

    const auto& tmNameDef = IndexData->GetRTYSearcherConfig()->TextMachine.DefaultTextMachine;
    const TString tmName = Cgi->GetTmName().GetOrElse(tmNameDef);
    TIntrusivePtr<ISaasTextMachinePlugin> tmCreator = ISaasTextMachinePlugin::TFactory::Construct(tmName);
    Y_ENSURE(tmCreator, "text machine is not registered by name: " << tmName);

    CHECK_WITH_LOG(RankingPass.RTYFactorMask != nullptr);
    //TODO(SAAS-5958): Uncomment the logic below to implement SAAS-5958
    //if (!tmCreator->IsExternalRelevanceCalcer() && RankingPass.RTYFactorMask && !RankingPass.RTYFactorMask->UsesWebTextMachine) {
    //    // Here we disable LingBoost even if we are given &pron=qbundleiter and &qbundle= , because none of TG_TEXT_MACHINE factors are used (SAAS-5958)
    //    return {};
    //}

    const bool isFastRankSecondPass = (RP.RequestMode & RPF_FASTRANK) && RankingPass.IsSecond;

    const NLingBoost::TRawConfig* rawConfig = tmCreator->GetRawConfig();
    const NLingBoost::TWordConfig* wordConfig = tmCreator->GetWordConfig();
    THolder<NLingBoost::TRawConfig> patchedRawConfig;

    if (RP.ReqBundleLimitsPatch) {
        const NSc::TValue scheme = NSc::TValue::FromJson(RP.ReqBundleLimitsPatch);
        patchedRawConfig.Reset(new NLingBoost::TRawConfig(*rawConfig));
        NLingBoost::PartialSchemeToConfig(scheme, *patchedRawConfig);
        rawConfig = patchedRawConfig.Get();
        // Word config is not owned by TWordHitsDispatcher, so we own it
        RankingPass.WordConfig.Reset(new NLingBoost::TWordConfig(*wordConfig));
        NLingBoost::PartialSchemeToConfig(scheme, *RankingPass.WordConfig);
        wordConfig = RankingPass.WordConfig.Get();
    }

    TVector<ui64> kps = Cgi->GetKps();
    if (kps.empty()) {
        kps.push_back(0);
    }
    TSaasTextMachineFeaturesCalcerFactory factory(tmCreator, *RP.RequestBundle, *rawConfig, BaseIndexData.GetReqBundleHashers(), std::move(kps));
    TTextMachineFeaturesCalcerFactoryBase::TFeatureOptions featOpts(*RP.ReqBundleParams);
    factory.InitTextMachine(MemPool, *wordConfig, featOpts);

    if (RP.ReqBundleLimitsMonitor) {
        factory.EnableLimitsMonitoring();
    }
    if (RP.NeedStreamHits) {
        factory.EnableHitsStorage();
    }

    const TAnnIndexManager* rtyAnn = IndexData->GetAnnManager();
    const bool allowAnnStreams = rtyAnn && !tmCreator->IsExternalRelevanceCalcer(); // by convention, all new text machines will read hits from Ann and FactorAnn, not only from TR.
    const bool allowFactorAnnStreams = allowAnnStreams && (!(RP.RequestMode & RPF_FASTRANK) || RankingPass.IsSecond);
    const NIndexAnn::TReaders* annReaders = allowAnnStreams && rtyAnn->HasType("ann") ? BaseIndexData.GetAnnReaders() : nullptr;
    const NIndexAnn::TReaders* factorAnnReaders = allowFactorAnnStreams && rtyAnn->HasType("factorann") ? IndexData->GetFactorAnnReaders(BaseIndexData) : nullptr;

    const TCateg relevCountry = IsCountry(RP.RelevRegion) ? RP.RelevRegion : RankingOpts.TmRelevCountry.GetOrElse(COUNTRY_RUSSIA);
    const TRTYIndexTextData& textOpts = IndexData->GetIndexTextOpts();
    if (needOpenIterators || isFastRankSecondPass) {
        factory.OpenTrIterator(BaseIndexData.GetTextYR(),
            BaseIndexData.GetLemmatizedLangMask(),
            BaseIndexData.GetFlatBastardsLangMask(),
            BaseIndexData.GetCustomTxtSentLenReader(),
            Makeup,
            textOpts.GetAutoTitleSentNum(),
            textOpts.GetTitleZonesMask());

        if (annReaders) {
            const NIndexAnn::TYndexReader* annKeyInv = annReaders->GetYndexReader();
            Y_ENSURE(annKeyInv);
            factory.OpenAnnIterator(annKeyInv->GetYndexRequester(),
                BaseIndexData.GetLemmatizedLangMask(),
                BaseIndexData.GetFlatBastardsLangMask(),
                relevCountry,
                annReaders->GetDocDataIndex(),
                annReaders->GetSentenceLengthsReader());
        }

        if (factorAnnReaders) {
            const NIndexAnn::TYndexReader* factorAnnKeyInv = factorAnnReaders->GetYndexReader();
            Y_ENSURE(factorAnnKeyInv);
            factory.OpenFactorAnnIterator(factorAnnKeyInv->GetYndexRequester(),
                relevCountry,
                factorAnnReaders->GetSentenceLengthsReader(),
                factorAnnReaders->GetDocDataIndex(),
                RP.FactorAnnGutFirstStageLimit);
        }
    } else {
        factory.InitTrHitsProviderData(BaseIndexData.GetCustomTxtSentLenReader(), Makeup, textOpts.GetAutoTitleSentNum(), textOpts.GetTitleZonesMask());
        if (annReaders)
            factory.InitAnnHitsProviderData(relevCountry, annReaders->GetDocDataIndex(), annReaders->GetSentenceLengthsReader());
    }

    THolder<TTextMachineFeaturesCalcer> machine = factory.CreateCalcer();
    if (tmCreator->IsExternalRelevanceCalcer()) {
        return TSaasTextMachineCalcerHolder(std::move(machine));
    } else {
        return TSaasTextMachineCalcerHolder(MakeIntrusive<TSaasTextMachineFeaturesCalcer>(std::move(machine), tmCreator->GetTargetSlice()));
    }
}

bool TRTYRelevance::ShouldDoublePruningTargetDocs() const {
    return RP.EnableDoublePruning.Get(RP.SearchType, BaseIndexData.GetSearchZoneString());
}
