///
/// Adapted from search/relevance/ann_features.cpp
///
#include "ann_features.h"

#include <kernel/factor_storage/factor_storage.h>
#include <kernel/groupattrs/metainfo.h>
#include <kernel/indexann_data/data.h>
#include <kernel/streams/streams_factors.h>
#include <kernel/hitinfo/hitinfo.h>

#include <search/relevance/tr_iterator_holder.h>

#include <ysite/yandex/posfilter/hits_loader.h>
#include <ysite/yandex/relevance/bclm.h>
#include <ysite/yandex/relevance/bocm.h>
#include <ysite/yandex/relevance/u_tracker.h>


//
// TRTYAllRegions
//
TRTYAllRegions::TRTYAllRegions()
    : Filter{{0, 0}}
{
}

TVector<TCateg> TRTYAllRegions::MakeAllRegions(const NGroupingAttrs::TMetainfo* geoa, TCateg relevGeo) {
    TVector<TCateg> allRegions;
    if (geoa) {
        for (TCateg now = relevGeo; now != END_CATEG; now = geoa->Categ2Parent(now)) {
            allRegions.push_back(now);
        }
    }
    return allRegions; // empty vector means "undefined", like RP.AllRegions
}

TRTYAllRegions::TRTYAllRegions(TCateg relevGeo, TCateg relevCountry, const TVector<TCateg>& allRegions) {
    TVector<TCateg> regionChain;
    regionChain.reserve(8);
    if (!allRegions.empty()) {
        // allRegions is built using geoa.c2p or explicitly provided in RP
        for (const TCateg now: allRegions) {
            regionChain.push_back(now);
            if (IsCountry(now)) {
                if (relevCountry != COUNTRY_INVALID && IsCountry(relevCountry)) {
                    if (relevCountry != now)
                        regionChain.push_back(relevCountry);
                    relevCountry = COUNTRY_INVALID;
                }
                break; // this drops regions bigger than IsCountry()
            }
        }
    } else {
        // empty allRegions means that there is no geobase and no RP.AllRegions.
        // We build a simplified regionChain - just relevGeo and relevCountry
        if (relevGeo != END_CATEG)
            regionChain.push_back(relevGeo);
        if (relevCountry != COUNTRY_INVALID && relevCountry != relevGeo)
            regionChain.push_back(relevCountry);
    }
    regionChain.push_back(0);

    Filter.resize(regionChain.size());
    ui32 cnt = 0;
    for (const TCateg now: regionChain) {
        Filter[cnt] = {now, cnt};
        ++cnt;
    }
    SortBy(Filter, [](const TRegionWithPrio& item) {
        return item.first;
    });
    auto eq = [](const TRegionWithPrio& a, const TRegionWithPrio& b) {
        return a.first == b.first;
    };
    Filter.erase(std::unique(Filter.begin(), Filter.end(), eq), Filter.end());
}

//
// TRTYQuorumAnnFeaturesCalcer
//
TRTYQuorumAnnFeaturesCalcer::TRTYQuorumAnnFeaturesCalcer(const IAnnDataReader* params, const TVector<float>& wordWeights, const TVector<TCateg>& allRegions, const TRTYRankingOpts& opts)
    : Params(params)
{
    if (Params && Params->GetWordWeights().size()) {
        NHopBclmTracker.Reset(new TBclmTracker(Params->GetWordWeights()));
        CorrCtrTracker.Init(Params->GetWordWeights());
    } else if (wordWeights.size()) {
        NHopBclmTracker.Reset(new TBclmTracker(wordWeights));
        CorrCtrTracker.Init(wordWeights);
    }
    if (Params && Params->GetDocDataIndex()) {
        AnnIter = Params->GetDocDataIndex()->CreateIterator();
    }

    if (opts.AnnRelevRegion.Defined()) {
        AnnRelevRegion = *opts.AnnRelevRegion;
    } else if (Params) {
        AnnRelevRegion = Params->GetRelevCountry();
    }

    RegionFilter = TRTYAllRegions(AnnRelevRegion, Params->GetRelevCountry(), allRegions);
}

TRTYQuorumAnnFeaturesCalcer::~TRTYQuorumAnnFeaturesCalcer() {
}

void TRTYQuorumAnnFeaturesCalcer::InitNextDoc(TNewDocParams* params) {
    if (!Params || !Params->GetDocDataIndex()) {
        return;
    }

    if (NHopBclmTracker.Get()) {
        NHopBclmTracker->NewDoc();
    }
    CorrCtrTracker.InitNextDoc();

    DocId = params->DocId;

    HasDoc = Params->GetDocDataIndex()->HasDoc(DocId);
    Params->GetDocOffsets(DocId, &Offsets);
}

void TRTYQuorumAnnFeaturesCalcer::AddPositions(TFullPositionEx* pos, size_t count, ERelevanceType /* rt */) {
    if (!HasDoc) {
        return;
    }

    for (size_t i = 0; i < count; ++i) {
        AddPosition(pos + i);
    }
}

void TRTYQuorumAnnFeaturesCalcer::AddPosition(const TFullPositionEx* pos) {
    const ui32 leftBreak = TWordPosition::Break(pos->Pos.Beg);

    ui32 breakLength = 0;
    if (Offsets.size() > (leftBreak + 1)) {
        breakLength = Offsets[leftBreak + 1] - Offsets[leftBreak];
    }

    bool nHopUsage = false;
    ui32 annData = 0;
    ui32 bestPrio = Max<ui32>();

    AnnIter->Restart(NIndexAnn::THitMask(DocId, leftBreak));
    for (const TRegionWithPrio& reg : RegionFilter.Filter) {
        ui32 region = reg.first;
        for (; AnnIter->Valid(); AnnIter->Next()) {
            const auto& cur = AnnIter->Current();
            if (cur.Region() < region) {
                continue;
            }
            if (cur.Region() > region) {
                break;
            }
            if (cur.Stream() == NIndexAnn::DT_NHOP) {
                if (cur.Region() == AnnRelevRegion) {
                    nHopUsage = true;
                }
            } else if (cur.Stream() == CorrCtrTracker.GetFieldId() && bestPrio > reg.second) {
                annData = cur.Value();
                bestPrio = reg.second;
            }
        }
        if (!AnnIter->Valid())
            break;
    }

    if (annData) {
        CorrCtrTracker.AddHit(
            leftBreak,
            breakLength,
            pos->Pos,
            pos->WordIdx,
            annData);
    }

    const ui32 rightBrk = TWordPosition::Break(pos->Pos.End);

    if (Params->IsOffsetsCorrect() && rightBrk < Offsets.size() && nHopUsage && NHopBclmTracker.Get()) {
        const ui32 rightPosition = Offsets[rightBrk] + TWordPosition::Word(pos->Pos.End);
        const ui32 leftPosition = Offsets[leftBreak] + TWordPosition::Word(pos->Pos.Beg);

        TFullPositionInfo info;
        info.Pos.Beg = pos->Pos.Beg;
        info.Pos.End = pos->Pos.End;
        info.Abs.Beg = leftPosition;
        info.Abs.End = rightPosition;

        NHopBclmTracker->Add(info, pos->WordIdx);
    }
}

void TRTYQuorumAnnFeaturesCalcer::CalcFeatures(TPosGatorCalcFeaturesParams& params) {
    if (!HasDoc) {
        return;
    }

    TAnnotationFactorsAccessor& factors = params.ProtectedFactor.GetAnnotationGroup();

    if (NHopBclmTracker.Get()) {
        float weightedScore = 0;
        float planeScore = 0;
        NHopBclmTracker->CalcResult(weightedScore, planeScore);
        factors.NHopTextBclmWeighted = weightedScore;
        factors.NHopTextBclmPlane = planeScore;
    }

    if (CorrCtrTracker.IsInited()) {
        const TUTracker* corrCtrRawUTracker = CorrCtrTracker.FinishDoc();
        factors.CorrectedCtrAnnotationMatchPrediction = corrCtrRawUTracker->CalcAnnotationMatchPrediction();
        factors.CorrectedCtrQueryMatchPrediction = corrCtrRawUTracker->CalcQueryMatchPrediction();
        factors.CorrectedCtrValueWcmAvg = corrCtrRawUTracker->CalcValueWcmAvg();
        factors.CorrectedCtrBm15V4K5 = corrCtrRawUTracker->CalcBm15V4(0.01f);
        factors.CorrectedCtrBm15StrictK2 = corrCtrRawUTracker->CalcBm15Strict(0.001f);
    }
}

//
// TRTYRegStreamValues
//
ui32 TRTYRegStreamValues::GetValue(NIndexAnn::EDataType dataType) const {
    Y_ASSERT(dataType < Values.size());
    return Values[dataType];
}

bool TRTYRegStreamValues::HasValue(NIndexAnn::EDataType dataType) const {
    return dataType < Inited.Size() && Inited.Test(dataType);
}

size_t TRTYRegStreamValues::Size() const {
    return Values.size();
}

bool TRTYRegStreamValues::Empty() const {
    return Inited.Empty();
}
void TRTYRegStreamValues::FillFrom(NIndexAnn::IDocDataIterator& iter, ui32 doc, ui32 breuk, const TVector<TRegionWithPrio>& regionChain) {
    Inited.Clear();
    iter.Restart(NIndexAnn::THitMask(doc, breuk));
    for (auto&& [region, prio] : regionChain) {
        for (; iter.Valid(); iter.Next()) {
            const auto& cur = iter.Current();
            if (cur.Region() < region) {
                continue;
            }
            if (cur.Region() > region) {
                break;
            }
            if (cur.Stream() < Values.size()) {
                if (!Inited.Test(cur.Stream()) || Prios[cur.Stream()] > prio) {
                    Prios[cur.Stream()] = prio;
                    Values[cur.Stream()] = cur.Value();
                }
                Inited.Set(cur.Stream());
            }
        }
        if (!iter.Valid())
            break;
    }
}

//
// TRTYFactorAnnFeaturesCalcer
//
class TRTYFactorAnnFeaturesRawCalcer {
public:
    TRTYFactorAnnFeaturesRawCalcer(const IAnnDataReader* params, const TRTYRankingOpts& rtyOpts, const TVector<TCateg>& allRegions);
    ~TRTYFactorAnnFeaturesRawCalcer();

    void InitNextDoc(ui32 docId);
    void AddPositions(TFullPositionEx* pos, size_t count);
    void CalcFeatures(TProtectedFactorStorage& factors);

private:
    const IAnnDataReader* Params;
    THolder<NIndexAnn::IDocDataIterator> AnnIter;
    bool HasDoc = false;
    ui32 DocId = 0;
    TRTYRegStreamValues StreamValues;
    TRTYAllRegions RegionFilter;
    TSentenceOffsets Offsets;
    TVector<TAutoPtr<TUTrackerWrapper>> Wrappers;

private:
    template <NIndexAnn::EDataType DataType, ui8 MaxCalcLevel = TUTrackerWrapper::MaxPossibleCalcLevel>
    void AddWrapper() {
        Y_ASSERT(Wrappers.size() > DataType);
        Wrappers[DataType].Reset(new TAnnUTrackerWrapper<DataType, MaxCalcLevel>);
        Wrappers[DataType]->Init(Params->GetWordWeights());
    }
};

TRTYFactorAnnFeaturesRawCalcer::TRTYFactorAnnFeaturesRawCalcer(const IAnnDataReader* params, const TRTYRankingOpts& rtyOpts, const TVector<TCateg>& allRegions)
    : Params(params)
    , RegionFilter(rtyOpts.AnnRelevRegion.GetOrElse(END_CATEG), params->GetRelevCountry(), allRegions)
{
    Y_VERIFY(Params);
    Wrappers.resize(NIndexAnn::DT_NUM_TYPES);
    if (Params->GetDocDataIndex()) {
        AnnIter = Params->GetDocDataIndex()->CreateIterator();
    }

    if (Params->GetWordWeights().size()) {
        AddWrapper<NIndexAnn::DT_SAMPLE_PERIOD, 1>();
        AddWrapper<NIndexAnn::DT_ONE_CLICK, 3>();
        AddWrapper<NIndexAnn::DT_LONG_CLICK, 3>();
        AddWrapper<NIndexAnn::DT_SPLIT_DT, 0>();
        AddWrapper<NIndexAnn::DT_BQPR, 0>();
        AddWrapper<NIndexAnn::DT_YABAR_VISITS, 0>();
        AddWrapper<NIndexAnn::DT_YABAR_TIME, 0>();
        AddWrapper<NIndexAnn::DT_SIMPLE_CLICK, 0>();
        AddWrapper<NIndexAnn::DT_RANDOM_LOG_DBM35, 0>();
        AddWrapper<NIndexAnn::DT_CORRECTED_CTR_XFACTOR, 0>();
        AddWrapper<NIndexAnn::DT_POPULAR_SE_FRC_BROWSER, 1>();
        AddWrapper<NIndexAnn::DT_LONG_CLICK_SP, 1>();
        AddWrapper<NIndexAnn::DT_BQPR_SAMPLE, 2>();
        AddWrapper<NIndexAnn::DT_DOUBLE_FRC, 1>();
        AddWrapper<NIndexAnn::DT_ONE_CLICK_FRC_XF_SP, 2>();
        AddWrapper<NIndexAnn::DT_QUERY_DWELL_TIME, 1>();
    }
}

TRTYFactorAnnFeaturesRawCalcer::~TRTYFactorAnnFeaturesRawCalcer() {
}

void TRTYFactorAnnFeaturesRawCalcer::InitNextDoc(ui32 docId) {
    if (!Params || !Params->GetDocDataIndex()) {
        return;
    }

    for (auto& wrapper : Wrappers) {
        if (wrapper.Get() != nullptr) {
            wrapper->InitNextDoc();
        }
    }
    DocId = docId;

    HasDoc = Params->GetDocDataIndex()->HasDoc(docId);
    Params->GetDocOffsets(docId, &Offsets);
}

void TRTYFactorAnnFeaturesRawCalcer::AddPositions(TFullPositionEx* pos, size_t count) {
    if (!HasDoc) {
        return;
    }

    const TFullPositionEx* begin = pos;
    const TFullPositionEx* end = pos + count;

    const TFullPositionEx* start = begin;
    const TFullPositionEx* finish = end;

    for (; start != end; start = finish) {
        const ui32 leftBreak = TWordPosition::Break(start->Pos.Beg);

        finish = start + 1;

        for (; finish != end && TWordPosition::Break(finish->Pos.Beg) == leftBreak; ++finish) {
        }

        StreamValues.FillFrom(*AnnIter, DocId, leftBreak, RegionFilter.Filter);
        if (StreamValues.Empty()) {
            continue;
        }

        ui32 breakLength;
        if (Offsets.size() > (leftBreak + 1)) {
            breakLength = Offsets[leftBreak + 1] - Offsets[leftBreak];
        } else {
            breakLength = 0;
        }

        const auto& flags = StreamValues.GetFlags();
        for (size_t i = flags.FirstNonZeroBit(); i != flags.Size(); i = flags.NextNonZeroBit(i)) {
            if (Wrappers[i]) {
                Wrappers[i]->AddSentence(leftBreak, breakLength, start, finish, StreamValues.GetValue(i));
            }
        }
    }
}

void TRTYFactorAnnFeaturesRawCalcer::CalcFeatures(TProtectedFactorStorage& pFactors) {
    if (!HasDoc) {
        return;
    }

    TAnnotationFactorsAccessor& factors = pFactors.GetAnnotationGroup();

    if (Wrappers[NIndexAnn::DT_SAMPLE_PERIOD]->IsInited()) {
        const auto tracker = Wrappers[NIndexAnn::DT_SAMPLE_PERIOD]->FinishDoc();
        factors.SamplePeriodDayFrcQueryMatchPrediction = tracker->CalcQueryMatchPrediction();
        factors.SamplePeriodDayFrcAnnotationMatchPrediction = tracker->CalcAnnotationMatchPrediction();
        factors.SamplePeriodDayFrcValueWcmAvg = tracker->CalcValueWcmAvg();
        factors.SamplePeriodDayFrcBm15MaxK3 = tracker->CalcBm15Max(0.01f);
        factors.SamplePeriodDayFrcBocmWeightedK3 = tracker->CalcBocmWeighted(0.0001f);
        factors.SamplePeriodDayFrcBocmDoubleK5 = tracker->CalcBocmDouble(0.01f);
    }

    if (Wrappers[NIndexAnn::DT_ONE_CLICK]->IsInited()) {
        const auto tracker = Wrappers[NIndexAnn::DT_ONE_CLICK]->FinishDoc();
        factors.OneClickQueryMatchPrediction = tracker->CalcQueryMatchPrediction();
        factors.OneClickAnnotationMatchPrediction = tracker->CalcAnnotationMatchPrediction();
        factors.OneClickAnnotationMatchPredictionWeighted = tracker->CalcAnnotationMatchPredictionWeighted();
        factors.OneClickBm15AK4 = tracker->CalcBm15VA(0.5f);
        factors.OneClickBocmWeightedW1K3 = tracker->CalcBocmWeightedW1(0.01f);
        factors.OneClickSynonymMatchPrediction = tracker->CalcSynonymMatchPrediction();
        factors.OneClickFullMatchPrediction = tracker->CalcFullMatchPrediction();
        factors.OneClickValueWcmAvg = tracker->CalcValueWcmAvg();
        factors.OneClickBocmWeightedMaxK1 = tracker->CalcBocmWeightedMax(0.0001f);
        factors.OneClickBm15StrictK2 = tracker->CalcBm15Strict(0.001f);
        factors.OneClickBm15MaxK3 = tracker->CalcBm15Max(0.01f);
        factors.OneClickBclmPlainW1K3 = tracker->CalcBclmPlainW1(0.001f);
        factors.OneClickValueWcmMax = tracker->CalcValueWcmMax();
        factors.OneClickValueWcmPrediction = tracker->CalcValueWcmPrediction();
        factors.OneClickBclmWeightedK3 = tracker->CalcBclmWeighted(0.01f);
    }

    if (Wrappers[NIndexAnn::DT_LONG_CLICK]->IsInited()) {
        const auto tracker = Wrappers[NIndexAnn::DT_LONG_CLICK]->FinishDoc();
        factors.LongClickQueryMatchPrediction = tracker->CalcQueryMatchPrediction();
        factors.LongClickAnnotationMatchPrediction = tracker->CalcAnnotationMatchPrediction();
        factors.LongClickAnnotationMatchPredictionWeighted = tracker->CalcAnnotationMatchPredictionWeighted();
        factors.LongClickBm15AK4 = tracker->CalcBm15VA(0.5f);
        factors.LongClickBocmWeightedW1K3 = tracker->CalcBocmWeightedW1(0.01f);
        factors.LongClickBocmPlain = tracker->CalcBocmPlain(0.01f);
    }

    if (Wrappers[NIndexAnn::DT_SPLIT_DT]->IsInited()) {
        const auto tracker = Wrappers[NIndexAnn::DT_SPLIT_DT]->FinishDoc();
        factors.SplitDwellTimeQueryMatchPrediction = tracker->CalcQueryMatchPrediction();
        factors.SplitDwellTimeAnnotationMatchPrediction = tracker->CalcAnnotationMatchPrediction();
        factors.SplitDwellTimeBocmWeightedMaxK1 = tracker->CalcBocmWeightedMax(0.0001f);
        factors.SplitDwellTimeFullMatchPrediction = tracker->CalcFullMatchPrediction();
        factors.SplitDwellTimeValueWcmAvg = tracker->CalcValueWcmAvg();
        factors.SplitDwellTimeBm15MaxK3 = tracker->CalcBm15Max(0.01f);
    }

    if (Wrappers[NIndexAnn::DT_BQPR]->IsInited()) {
        const auto tracker = Wrappers[NIndexAnn::DT_BQPR]->FinishDoc();
        factors.BQPRQueryMatchPrediction = tracker->CalcQueryMatchPrediction();
        factors.BQPRAnnotationMatchPrediction = tracker->CalcAnnotationMatchPrediction();
        factors.BQPRBocmWeightedW1K3 = tracker->CalcBocmWeightedW1(0.01f);
        factors.BQPRBm15StrictK2 = tracker->CalcBm15Strict(0.001f);
    }

    if (Wrappers[NIndexAnn::DT_YABAR_VISITS]->IsInited()) {
        const auto tracker = Wrappers[NIndexAnn::DT_YABAR_VISITS]->FinishDoc();
        factors.YabarVisitsQueryMatchPrediction = tracker->CalcQueryMatchPrediction();
        factors.YabarVisitsAnnotationMatchPrediction = tracker->CalcAnnotationMatchPrediction();
    }

    if (Wrappers[NIndexAnn::DT_YABAR_TIME]->IsInited()) {
        const auto tracker = Wrappers[NIndexAnn::DT_YABAR_TIME]->FinishDoc();
        factors.YabarTimeQueryMatchPrediction = tracker->CalcQueryMatchPrediction();
        factors.YabarTimeAnnotationMatchPrediction = tracker->CalcAnnotationMatchPrediction();
        factors.YabarTimeAnnotationMatchPredictionWeighted = tracker->CalcAnnotationMatchPredictionWeighted();
    }

    if (Wrappers[NIndexAnn::DT_SIMPLE_CLICK]->IsInited()) {
        const auto tracker = Wrappers[NIndexAnn::DT_SIMPLE_CLICK]->FinishDoc();
        factors.SimpleClickQueryMatchPrediction = tracker->CalcQueryMatchPrediction();
        factors.SimpleClickAnnotationMatchPrediction = tracker->CalcAnnotationMatchPrediction();
        factors.SimpleClickBclmWeightedK3 = tracker->CalcBclmWeighted(0.01f);
    }

    if (Wrappers[NIndexAnn::DT_RANDOM_LOG_DBM35]->IsInited()) {
        const auto tracker = Wrappers[NIndexAnn::DT_RANDOM_LOG_DBM35]->FinishDoc();
        factors.RandomLogDBM35XfactorFullMatchPrediction = tracker->CalcFullMatchPrediction();
        factors.RandomLogDBM35XfactorAnnotationMatchPrediction = tracker->CalcAnnotationMatchPrediction();
        factors.RandomLogDBM35XfactorValueWcmMax = tracker->CalcValueWcmMax();
        factors.RandomLogDBM35XfactorValueWcmAvg = tracker->CalcValueWcmAvg();
        factors.RandomLogDBM35XfactorBm15StrictK2 = tracker->CalcBm15Strict(0.001f);
        factors.RandomLogDBM35XfactorBclmPlainW1K3 = tracker->CalcBclmPlainW1(0.001f);
        factors.RandomLogDBM35XfactorBclmWeightedK3 = tracker->CalcBclmWeighted(0.01f);
        factors.RandomLogDBM35XfactorBocmWeightedW1K3 = tracker->CalcBocmWeightedW1(0.01f);
    }

    if (Wrappers[NIndexAnn::DT_CORRECTED_CTR_XFACTOR]->IsInited()) {
        const auto tracker = Wrappers[NIndexAnn::DT_CORRECTED_CTR_XFACTOR]->FinishDoc();
        factors.CorrectedCtrXfactorAnnotationMatchPrediction = tracker->CalcAnnotationMatchPrediction();
        factors.CorrectedCtrXfactorQueryMatchPrediction = tracker->CalcQueryMatchPrediction();
        factors.CorrectedCtrXfactorValueWcmMax = tracker->CalcValueWcmMax();
        factors.CorrectedCtrXfactorValueWcmAvg = tracker->CalcValueWcmAvg();
        factors.CorrectedCtrXfactorBocmWeightedW1K3 = tracker->CalcBocmWeightedW1(0.01f);
        factors.CorrectedCtrXfactorBclmPlainK3 = tracker->CalcBclmPlain(0.0001f);
        factors.CorrectedCtrXfactorBclmMixPlainW1K1 = tracker->CalcBclmMixPlainW1(0.00001f, 0.01f);
    }

    if (Wrappers[NIndexAnn::DT_POPULAR_SE_FRC_BROWSER]->IsInited()) {
        const auto tracker = Wrappers[NIndexAnn::DT_POPULAR_SE_FRC_BROWSER]->FinishDoc();
        factors.PopularSEFRCBrowserAnnotationMatchPrediction = tracker->CalcAnnotationMatchPrediction();
        factors.PopularSEFRCBrowserSynonymMatchPrediction = tracker->CalcSynonymMatchPrediction();
        factors.PopularSEFRCBrowserValueWcmPrediction = tracker->CalcValueWcmPrediction();
        factors.PopularSEFRCBrowserBclmWeightedV2K3 = tracker->CalcBclmWeightedV2(0.01f);
        factors.PopularSEFRCBrowserBclmMixPlainW1K1 = tracker->CalcBclmMixPlainW1(0.0001f, 0.01f);
    }

    if (Wrappers[NIndexAnn::DT_LONG_CLICK_SP]->IsInited()) {
        const auto tracker = Wrappers[NIndexAnn::DT_LONG_CLICK_SP]->FinishDoc();
        factors.LongClickSPBclmPlainK5 = tracker->CalcBclmPlain(0.01f);
        factors.LongClickSPValueWcmPrediction = tracker->CalcValueWcmPrediction();
        factors.LongClickSPQueryMatchPrediction = tracker->CalcQueryMatchPrediction();
        factors.LongClickSPAnnotationMatchPrediction = tracker->CalcAnnotationMatchPrediction();
        factors.LongClickSPBclmPlainW1K3 = tracker->CalcBclmPlainW1(0.001f);
        factors.LongClickSPValueWcmAvg = tracker->CalcValueWcmAvg();
        factors.LongClickSPBm15StrictK2 = tracker->CalcBm15Strict(0.001f);
        factors.LongClickSPBocmDoubleK5 = tracker->CalcBocmDouble(0.01f);
        factors.LongClickSPBclmWeightedK3 = tracker->CalcBclmWeighted(0.01f);
        factors.LongClickSPBclmWeightedV2K3 = tracker->CalcBclmWeightedV2(0.01f);
        factors.LongClickSPBocmWeightedW1K3 = tracker->CalcBocmWeightedW1(0.01f);
    }

    if (Wrappers[NIndexAnn::DT_BQPR_SAMPLE]->IsInited()) {
        const auto tracker = Wrappers[NIndexAnn::DT_BQPR_SAMPLE]->FinishDoc();
        factors.BQPRSampleWcmCoverageMax = tracker->CalcWcmCoverageMax();
        factors.BQPRSampleFullMatchPrediction = tracker->CalcFullMatchPrediction();
        factors.BQPRSampleAnnotationMatchPredictionWeighted = tracker->CalcAnnotationMatchPrediction();
        factors.BQPRSampleValuePcmAvg = tracker->CalcValuePcmAvg();
        factors.BQPRSampleValueWcmAvg = tracker->CalcValueWcmAvg();
        factors.BQPRSampleBm15V4K8 = tracker->CalcBm15V4(10.f);                 //Bm15V4K8
        factors.BQPRSampleBocmWeightedV4K8 = tracker->CalcBocmWeightedV4(10.f); //BocmWeightedV4K8
        factors.BQPRSampleWcmMax = tracker->CalcWcmMax();
        factors.BQPRSampleSynonymMatchPrediction = tracker->CalcSynonymMatchPrediction();
        factors.BQPRSampleAnnotationMatchPrediction = tracker->CalcAnnotationMatchPrediction();
        factors.BQPRSampleSuffixMatchCount = tracker->CalcSuffixMatchCount();
        factors.BQPRSampleWcmCoveragePrediction = tracker->CalcWcmCoveragePrediction();
    }

    if (Wrappers[NIndexAnn::DT_DOUBLE_FRC]->IsInited()) {
        const auto tracker = Wrappers[NIndexAnn::DT_DOUBLE_FRC]->FinishDoc();
        factors.DoubleFrcFullMatchPrediction = tracker->CalcFullMatchPrediction();
        factors.DoubleFrcSynonymMatchPrediction = tracker->CalcSynonymMatchPrediction();
        factors.DoubleFrcAnnotationMatchPrediction = tracker->CalcAnnotationMatchPrediction();
        factors.DoubleFrcAnnotationMatchPredictionWeighted = tracker->CalcAnnotationMatchPredictionWeighted();
        factors.DoubleFrcQueryMatchPrediction = tracker->CalcQueryMatchPrediction();
        factors.DoubleFrcValueWcmAvg = tracker->CalcValueWcmAvg();
        factors.DoubleFrcBocmWeightedMaxK1 = tracker->CalcBocmWeightedMax(0.0001f);
        factors.DoubleFrcBm15V4K5 = tracker->CalcBm15V4(0.01f);
        factors.DoubleFrcBocmWeightedV4K5 = tracker->CalcBocmWeightedV4(0.01f);
        factors.DoubleFrcBocmDoubleK1 = tracker->CalcBocmDouble(0.000001f);
    }

    if (Wrappers[NIndexAnn::DT_ONE_CLICK_FRC_XF_SP]->IsInited()) {
        const auto tracker = Wrappers[NIndexAnn::DT_ONE_CLICK_FRC_XF_SP]->FinishDoc();
        factors.OneClickFrcXfSpFullMatchPrediction = tracker->CalcFullMatchPrediction();
        factors.OneClickFrcXfSpAnnotationMatchPredictionWeighted = tracker->CalcAnnotationMatchPredictionWeighted();
        factors.OneClickFrcXfSpValueWcmAvg = tracker->CalcValueWcmAvg();
        factors.OneClickFrcXfSpWcmMax = tracker->CalcWcmMax();
        factors.OneClickFrcXfSpWcmCoveragePrediction = tracker->CalcWcmCoveragePrediction();
        factors.OneClickFrcXfSpWcmCoverageMax = tracker->CalcWcmCoverageMax();
        factors.OneClickFrcXfSpPcmMax = tracker->CalcPcmMax();
        factors.OneClickFrcXfSpPrefixMatchCount = tracker->CalcPrefixMatchCount();
        factors.OneClickFrcXfSpSuffixMatchCount = tracker->CalcSuffixMatchCount();
        factors.OneClickFrcXfSpBm15V0W1K1 = tracker->CalcBm15V0W1(0.001f);
    }

    if (Wrappers[NIndexAnn::DT_QUERY_DWELL_TIME]->IsInited()) {
        const auto tracker = Wrappers[NIndexAnn::DT_QUERY_DWELL_TIME]->FinishDoc();
        factors.QueryDwellTimeFullMatchPrediction = tracker->CalcFullMatchPrediction();
        factors.QueryDwellTimeSynonymMatchPrediction = tracker->CalcSynonymMatchPrediction();
        factors.QueryDwellTimeAnnotationMatchPrediction = tracker->CalcAnnotationMatchPrediction();
        factors.QueryDwellTimeAnnotationMatchPredictionWeighted = tracker->CalcAnnotationMatchPredictionWeighted();
        factors.QueryDwellTimeQueryMatchPrediction = tracker->CalcQueryMatchPrediction();
        factors.QueryDwellTimeValueWcmAvg = tracker->CalcValueWcmAvg();
        factors.QueryDwellTimeBclmPlainW1K3 = tracker->CalcBclmPlainW1(0.001f);
        factors.QueryDwellTimeBm15CoverageV4K3 = tracker->CalcBm15CoverageV4(0.01f);
        factors.QueryDwellTimeBclmPlainK4 = tracker->CalcBclmPlain(0.001f);
        factors.QueryDwellTimeBocmWeightedV4K5 = tracker->CalcBocmWeightedV4(0.01f);
    }
}

class TRTYFactorAnnFeaturesCalcer::TImpl {
private:
    THolder<THitsLoader> AnnHitsLoader;
    THolder<ITRIterator> AnnTrIterator;
    THolder<TDocIdIterator> AnnDocIdIterator;
    THolder<TNextDocSwitcher> AnnSwitcher;
    TNextDocSwitcher::TResult AnnSwitchResult;
    TAnnFeaturesParams AnnParams;
    THolder<TRTYFactorAnnFeaturesRawCalcer> AnnCalcer;
    TVector<TFullPositionEx> AnnPositions;

public:
    void ResetRequest(const NIndexAnn::TReaders& readers,
                      const TRichRequestNode& root,
                      const TVector<TCateg>& allRegions,
                      const TRTYRankingOpts& rtyOpts) {
        // it is better to crash explicitly here than crash at corrupted object method call
        Y_VERIFY(readers.GetYndexReader() != nullptr, "Ann yndex reader should be initialized. ");
        Y_VERIFY(readers.GetDocDataIndex() != nullptr, "Ann data reader should be initialized. ");
        Y_VERIFY(readers.GetSentenceLengthsReader() != nullptr, "Ann sent reader should be initialized. ");

        const TYndexRequester* yr = readers.GetYndexReader()->GetYndexRequester();

        AnnHitsLoader.Reset(new THitsLoader(
            &yr->YMain(),
            THitsLoader::GetDefaultLemmatizedLangMask(),
            THitsLoader::GetDefaultFlatBastardsLangMask(),
            yr->GetAnchorWordWeight(),
            yr->GetHitsForReadLoader(),
            THitsLoader::IDX_ANN,
            true));

        AnnTrIterator.Reset(
            CreateTRIteratorSimple(
                AnnHitsLoader.Get(),
                AnnHitsLoader.Get(),
                root)
                .Release());

        if (AnnTrIterator.Get()) {
            AnnDocIdIterator.Reset(new TDocIdIterator);
            AnnSwitcher.Reset(new TNextDocSwitcher(
                AnnTrIterator.Get(),
                nullptr /*reqsIterArg*/,
                nullptr /*filteringViewer*/,
                nullptr /*StrictPruningFilter*/,
                nullptr /*pruningFilter */,
                nullptr /*pruning*/,
                FINAL_DOC_BITS /*upperBound*/,
                AnnDocIdIterator.Get()));

            AnnParams.Iterator = AnnTrIterator.Get();
            AnnParams.RelevCountry = rtyOpts.TmRelevCountry.GetOrElse(COUNTRY_INVALID);

            AnnParams.SetReaders(readers);
            AnnParams.OpenIndexAnnDataIterator();
            AnnParams.LoadWordWeights();

            AnnCalcer.Reset(new TRTYFactorAnnFeaturesRawCalcer(&AnnParams, rtyOpts, !rtyOpts.UseRegionChain ? TVector<TCateg>() : allRegions));

            const size_t maxPositions = Y_UNLIKELY(rtyOpts.FactorAnnHitsLimit)
                ? rtyOpts.FactorAnnHitsLimit
                : TFactorAnnFeaturesCalcer::NUM_HIT_POSITIONS;
            AnnPositions.resize(maxPositions);
        }
    }

    void CalcDocFactors(ui32 docId, TProtectedFactorStorage& factors) {
        Y_ASSERT(AnnCalcer != nullptr);
        if (AnnTrIterator.Get() && AnnCalcer.Get()) {
            // This should guarantee quorum disabling.
            AnnDocIdIterator->Add(docId);
            AnnDocIdIterator->Next(TWordPosition(docId, 0).Pos);

            AnnSwitchResult.DocId = docId;
            const bool smthFound = AnnSwitcher->Switch(&AnnSwitchResult);

            if (smthFound && AnnSwitchResult.DocId == docId && AnnSwitchResult.HasTRpos) {
                const size_t numAnnPos = AnnTrIterator->GetPositions(docId, &AnnPositions[0], AnnPositions.size());

                if (numAnnPos > 0 && AnnCalcer) {
                    AnnCalcer->InitNextDoc(docId);
                    AnnCalcer->AddPositions(&AnnPositions[0], numAnnPos);
                    AnnCalcer->CalcFeatures(factors);
                }
            }
        }
    }
};

TRTYFactorAnnFeaturesCalcer::TRTYFactorAnnFeaturesCalcer()
    : Impl(new TImpl)
    , ViewMain(0)
{
}

TRTYFactorAnnFeaturesCalcer::~TRTYFactorAnnFeaturesCalcer() {
}

void TRTYFactorAnnFeaturesCalcer::ResetRequest(const NIndexAnn::TReaders& readers,
                                               const TRichRequestNode& root,
                                               const TVector<TCateg>& allRegions,
                                               const TRTYRankingOpts& rankingOpts) {
    Impl->ResetRequest(readers, root, allRegions, rankingOpts);
}

void TRTYFactorAnnFeaturesCalcer::Bind(TFactorStorage& storage) {
    ViewMain = storage.CreateViewFor(EFactorSlice::WEB_PRODUCTION);
}

void TRTYFactorAnnFeaturesCalcer::CalcFeatures(ui32 docId) {
    TProtectedFactorStorage pfactors(ViewMain);
    Impl->CalcDocFactors(docId, pfactors);
}

