#include "user_factors_calcer.h"
#include "function.h"
#include <saas/util/logging/exception_process.h>
#include <library/cpp/object_factory/object_factory.h>
#include <library/cpp/string_utils/base64/base64.h>
#include <kernel/dssm_applier/begemot/production_data.h>
#include <kernel/searchlog/searchlog.h>
#include <kernel/dssm_applier/utils/utils.h>
#include <kernel/dssm_applier/embeddings_transfer/proto/model_embedding.pb.h>
#include <kernel/dssm_applier/embeddings_transfer/embeddings_transfer.h>

// can't find argsort in library
#include <ads/quality/trafarets/libs/generic/algo/argsort/argsort.h>

#include <util/string/cast.h>
#include <util/digest/city.h>
#include <util/digest/fnv.h>
#include <array>
#include <limits>

using TImportedFunctions = NRTYFeatures::TImportedFunctions;

namespace NRTYFactors {
    namespace NCalcers {

        TUserFactorsCalcer::IFactorCalcer* CreateCalcerFromString(const TStringBuf& src, size_t& pos, const TInitCalcerParams& params);

        inline bool SkipSpaces(const TStringBuf& src, size_t& pos) {
            while(pos < src.size() && src[pos] == ' ') ++pos;
            return pos < src.size();
        }

        class TFactorCalcerConst : public TUserFactorsCalcer::IFactorCalcer {

            void ParseNumber(const TStringBuf& src, size_t& pos) {
                const size_t endpos = src.find_first_of(",)", pos);
                const TStringBuf sub = endpos != TStringBuf::npos ? src.substr(pos, endpos - pos) : src.substr(pos);

                float floatValue;
                if (TryFromString(sub, floatValue)) {
                    FloatValue = floatValue;
                }

                i64 intValue;
                if (TryFromString(sub, intValue)) {
                    IntValue = intValue;
                }

                pos = endpos;
            }

            void ParseString(const TStringBuf& src, size_t& pos) {
                bool lastQuoteIsFound = false;
                TStringStream value;
                size_t endpos = pos + 1;
                for (; endpos < src.size(); endpos++) {
                    if (src[endpos] == '"') {
                        lastQuoteIsFound = true;
                        break;
                    }
                    if (src[endpos] == '\\') {
                        Y_ENSURE(endpos + 1 < src.size(), "Incorrect string literal, no character after escaping symbol");
                        endpos++;

                        switch (src[endpos]) {
                            case '"':
                                value << '"';
                                break;
                            case '\\':
                                value << '\\';
                                break;
                            case 't':
                                value << '\t';
                                break;
                            default:
                                Y_ENSURE(false, "Incorrect symbol escape");
                                break;
                        }
                        continue;
                    }

                    value << src[endpos];
                }
                Y_ENSURE(lastQuoteIsFound, "Incorrect string literal, closing quote is not found");

                StringValue = value.Str();
                pos = endpos + 1;
            }
        public:
            TFactorCalcerConst(const TStringBuf& src, size_t& pos) {
                if (src[pos] == '"') {
                    ParseString(src, pos);
                } else {
                    ParseNumber(src, pos);
                }
            }

            void FillUsedFactors(TUserFactorsCalcer::TIndexSet& /*indexes*/) const override {
            }

            TStringBuf GetCalculatorName() const override {
                return "const_function";
            }

            float DoCalc(TCalcFactorsContext& /*ctx*/) const override {
                if (Y_LIKELY(FloatValue.Defined())) {
                    return FloatValue.GetRef();
                }
                ythrow yexception() << "Incorrect float сonstant in calc= expression";
            }

            i64 DoCalcInt(TCalcFactorsContext& /*ctx*/) const override {
                if (Y_LIKELY(IntValue.Defined())) {
                    return IntValue.GetRef();
                }
                ythrow yexception() << "Incorrect integer сonstant in calc= expression";
            }

            TString DoCalcBlob(TCalcFactorsContext& /*ctx*/) const override {
                if (Y_LIKELY(StringValue.Defined())) {
                    return StringValue.GetRef();
                }
                ythrow yexception() << "Incorrect string сonstant in calc= expression";
            }

        private:
            TMaybe<float> FloatValue;
            TMaybe<i64> IntValue;
            TMaybe<TString> StringValue;
        };

        class TFactorCalcerFactor : public TUserFactorsCalcer::IFactorCalcer {
        public:
            TFactorCalcerFactor(const TStringBuf& src, size_t& pos, const TConfig& factorConfig)
                : Index(NOT_FACTOR)
            {
                SetDependsOnDoc();
                ++pos;
                const size_t endpos = src.find_first_of(" ,)", pos);
                Index = factorConfig.GetFactorGlobalNum(src.data() + pos, (endpos != TStringBuf::npos) ? (endpos - pos) : TStringBuf::npos);
                if(Index == NOT_FACTOR)
                    ythrow yexception() << "Unknown factor: #" << src.substr(pos);
                pos = endpos;
            }

            float DoCalc(TCalcFactorsContext& ctx) const override {
                return (*ctx.Factors)[Index];

            }

            void Init(TUserFactorsCalcer::TCalcers& calcers) override {
                TUserFactorsCalcer::TCalcers::iterator i = calcers.find(Index);
                if (i != calcers.end())
                    Factor = i->second.GetCalcer();
            }

            void FillUsedFactors(TUserFactorsCalcer::TIndexSet& indexes) const override {
                indexes.insert(Index);
                if (!!Factor)
                    Factor->FillUsedFactors(indexes);
            }

        private:
            size_t Index;
            TUserFactorsCalcer::TFactorCalcerPtr Factor;
        };

        class TFactorCalcerGroupAttr : public TUserFactorsCalcer::IFactorCalcer {
        public:
            static const TString Prefix;
            TFactorCalcerGroupAttr(const TStringBuf& src, size_t& pos, const NGroupingAttrs::TDocsAttrs& docsAttrs)
                : DocsAttrs(docsAttrs)
            {
                SetDependsOnDoc();
                pos += Prefix.length();
                size_t endpos = src.find_first_of(" ,)", pos);
                if (endpos == TStringBuf::npos)
                    endpos = src.size();
                AttrName = TString(src.data() + pos, endpos - pos);
                AttrNum = DocsAttrs.Config().AttrNum(AttrName.data());
                pos = endpos;
            }

            void FillUsedFactors(TUserFactorsCalcer::TIndexSet& /*indexes*/) const override {
            }

            TString DescribeAs(EExprType t) const override {
                if (t == EExprType::GroupingAttr) {
                    return AttrName;
                }
                return Default<TString>();
            }

            float DoCalc(TCalcFactorsContext& ctx) const override {
                TCategSeries categs;
                if (AttrNum == NGroupingAttrs::TConfig::NotFound || !DocsAttrs.DocCategs(ctx.DocId, AttrNum, categs))
                    return 0.f;
                if (categs.size() != 1)
                    return 0.f;
                return categs.GetCateg(0);
            }

        private:
            const NGroupingAttrs::TDocsAttrs& DocsAttrs;
            TString AttrName;
            ui32 AttrNum;
        };

        const TString TFactorCalcerGroupAttr::Prefix("#group_");


        // TFactorCalcerGeoLayer: syntaxic sugar that allows referring to a rty_mingeo layer like '#geo_delivery_zone'
        // The return value is an integer constant, which is defined in relev.conf
        class TFactorCalcerGeoLayer : public TUserFactorsCalcer::IFactorCalcer {
        public:
            TFactorCalcerGeoLayer(const TStringBuf& src, size_t& pos, const TConfig& factorConfig)
                : LayerId(-1)
            {
                pos += Prefix.length();
                size_t endpos = src.find_first_of(" ,)", pos);
                if (endpos == TStringBuf::npos)
                    endpos = src.size();
                TString name(src.data() + pos, endpos - pos);
                size_t layerId = 0; // default for coords
                if (!factorConfig.GetStreamIndex(name, &layerId))
                    ythrow yexception() << "unknown stream: " << name;
                Y_ENSURE(layerId < Max<ui8>(), "incorrect id for " << name);
                LayerId = static_cast<ui8>(layerId);
                pos = endpos;
            }

            i64 DoCalcInt(TCalcFactorsContext& /*ctx*/) const override {
                return LayerId;
            }

            void FillUsedFactors(TUserFactorsCalcer::TIndexSet& /*indexes*/) const override {
            }

            TString DescribeAs(EExprType t) const override {
                if (t == EExprType::SecondaryKey) {
                    return "-"; // a non zero string
                }
                return Default<TString>();
            }

        private:
            i64 LayerId;

        public:
            static const TString Prefix;
        };
        const TString TFactorCalcerGeoLayer::Prefix("#geo_");

        void TUserFunction::Init(const TStringBuf& src, const TStringBuf& name, size_t& pos, const TInitCalcerParams& params) {
            Args.clear();
            DDKManager = params.DDKManager;
            RP = params.RP;
            SearchStatistics = &params.searchStatistics;
            ValuesStorage = params.ValuesStorage;
            UserDefinedAttrs = params.UserDefinedAttrs;

            bool complete = src[pos] == ')';
            while (!complete) {
                Args.push_back(CreateCalcerFromString(src, pos, params));
                if (!Args.back() || !SkipSpaces(src, pos)) {
                    ythrow yexception() << "Error in arguments of function " << name;
                }
                switch (src[pos]) {
                case ')': complete = true; [[fallthrough]]; // AUTOGENERATED_FALLTHROUGH_FIXME
                case ',': ++pos; break;
                default:
                    ythrow yexception() << "Unexpected symbol " << (src.data() + pos);
                }
            }
        }

        void TUserFunction::Validate(const TStringBuf& /*name*/) {
        }

        void TFactorCalcerFunc::UpdateDependsOnDoc() const {
            for (const auto& arg : Args) {
                if (arg->DependsOnDoc()) {
                    SetDependsOnDoc();
                }
            }
        }

        void TFactorCalcerFunc::FillUsedFactors(TUserFactorsCalcer::TIndexSet& indexes) const {
            for (TArgs::const_iterator i = Args.begin(); i != Args.end(); ++i)
                (*i)->FillUsedFactors(indexes);
        }

        void TStoreValueFunc::UpdateDependsOnDoc() {
            for (const auto& arg : Args) {
                if (arg->DependsOnDoc()) {
                    SetDependsOnDoc();
                }
            }
        }

        void TStoreValueFunc::Init(const TStringBuf& src, const TStringBuf& name, size_t& pos, const TInitCalcerParams& params) {
            TUserFunction::Init(src, name, pos, params);
        }

        class TFactorCalcerInSet : public TFactorCalcerFunc {
        public:
            float DoCalc(TCalcFactorsContext& ctx) const override {
                float val = Args[0]->Calc(ctx);
                for (TArgs::const_iterator i = Args.begin() + 1; i != Args.end(); ++i)
                    if (fabs((*i)->Calc(ctx) - val) < Epsilon)
                        return 1.f;
                return 0.f;
            }

            void Validate(const TStringBuf& name) override {
                if (Args.size() < 2)
                    ythrow yexception() << "function '" << name << "' must have at least 1 argument";
            }

            static constexpr float Epsilon = 1e-6f;

        private:
            static TFuncFactory::TRegistrator<TFactorCalcerInSet> Registrator;
        };

        TFuncFactory::TRegistrator<TFactorCalcerInSet> TFactorCalcerInSet::Registrator("inset");

        class TFactorCalcerInSetAny : public TFactorCalcerInSet {
        private:
            const NGroupingAttrs::TDocsAttrs* DocsAttrs = nullptr;
            ui32 AttrNum;

        private:
            bool IsGroupAttrMode() const {
                return DocsAttrs != nullptr;
            };

        public:
            float DoCalc(TCalcFactorsContext& ctx) const override {
                if (!IsGroupAttrMode()) {
                    // This branch is called when Arg[0] subexression is anything but simple "#group_xxxx"
                    return TFactorCalcerInSet::DoCalc(ctx);
                }

                Y_ASSERT(DocsAttrs);
                TCategSeries categs;
                if (DocsAttrs->DocCategs(ctx.DocId, AttrNum, categs)) {
                    for (TArgs::const_iterator i = Args.begin() + 1; i != Args.end(); ++i) {
                        float argVl = (*i)->Calc(ctx);
                        for (auto iCateg = categs.Begin(); iCateg != categs.End(); ++iCateg) {
                            float vl = *iCateg;
                            if (fabs(argVl - vl) < Epsilon)
                                return 1.f;
                        }
                    }
                }
                return 0.f;
            }

            using TFactorCalcerInSet::Init;

            void Init(const TStringBuf& src, const TStringBuf& name, size_t& pos, const TInitCalcerParams& params) override {
                TFactorCalcerInSet::Init(src, name, pos, params);
                const TString attrName = Args[0]->DescribeAs(EExprType::GroupingAttr);
                if (attrName) {
                    Y_ENSURE(params.DocsAttrs, "there is no grouping attrs");
                    ui32 attrNum = params.DocsAttrs->Config().AttrNum(attrName.data());
                    if (attrNum != NGroupingAttrs::TConfig::NotFound) {
                        DocsAttrs = params.DocsAttrs; // enters IsGroupAttrMode() mode
                        AttrNum = attrNum;
                    }
                }
            }

        private:
            static TFuncFactory::TRegistrator<TFactorCalcerInSetAny> Registrator;
        };

        TFuncFactory::TRegistrator<TFactorCalcerInSetAny> TFactorCalcerInSetAny::Registrator("insetany");

        class TFactorCalcerIf : public TFactorCalcerFunc {
        public:
            float DoCalc(TCalcFactorsContext& ctx) const override {
                if (Args[0]->Calc(ctx) > 1e-6)
                    return Args.size() > 1 ? Args[1]->Calc(ctx) : 1.f;
                return Args.size() > 2 ? Args[2]->Calc(ctx) : 0.f;
            }

            void Validate(const TStringBuf&) override {
                if (Args.size() < 1 || Args.size() > 3)
                    ythrow yexception() << "function 'if' must have from 1 to 3 arguments";
            }

            static TFuncFactory::TRegistrator<TFactorCalcerIf> Registrator;
        };

        TFuncFactory::TRegistrator<TFactorCalcerIf> TFactorCalcerIf::Registrator("if");

        class TFactorCalcerGt : public TFactorCalcerFunc {
        public:
            float DoCalc(TCalcFactorsContext& ctx) const override{
                return Args[0]->Calc(ctx) > Args[1]->Calc(ctx) ? 1.f : 0.f;
            }

            void Validate(const TStringBuf& name) override {
                if (Args.size() != 2)
                    ythrow yexception() << "function '" << name << "' must have 2 arguments";
            }

            static TFuncFactory::TRegistrator<TFactorCalcerGt> Registrator;
        };

        TFuncFactory::TRegistrator<TFactorCalcerGt> TFactorCalcerGt::Registrator("gt");

        class TFactorCalcerLt : public TFactorCalcerGt {
        public:
            float DoCalc(TCalcFactorsContext& ctx) const override {
                return Args[0]->Calc(ctx) < Args[1]->Calc(ctx) ? 1.f : 0.f;
            }

            static TFuncFactory::TRegistrator<TFactorCalcerLt> Registrator;
        };

        TFuncFactory::TRegistrator<TFactorCalcerLt> TFactorCalcerLt::Registrator("lt");

        class TFactorCalcerDiff : public TFactorCalcerFunc {
        public:
            float DoCalc(TCalcFactorsContext& ctx) const override {
                return Args[0]->Calc(ctx) - Args[1]->Calc(ctx);
            }

            void Validate(const TStringBuf&) override {
                if (Args.size() != 2)
                    ythrow yexception() << "function 'diff' must have exactly 2 arguments";
            }

            static TFuncFactory::TRegistrator<TFactorCalcerDiff> Registrator;
        };

        TFuncFactory::TRegistrator<TFactorCalcerDiff> TFactorCalcerDiff::Registrator("diff");

        class TFactorCalcerHash : public TFactorCalcerFunc {
            static constexpr int MAX_ARGS_COUNT = 100;

            float DoCalc(TCalcFactorsContext& ctx) const override {
                std::array<i64, MAX_ARGS_COUNT> buffer;
                for (size_t argIdx = 0; argIdx < Args.size(); argIdx++) {
                    buffer[argIdx] = Args[argIdx]->CalcInt(ctx);
                }

                const TArrayRef<const char> bytesArray = as_bytes(TArrayRef<i64>(buffer.data(), Args.size()));
                double result = ::FnvHash<ui64>(bytesArray.begin(), bytesArray.end());
                result /= std::numeric_limits<ui64>::max();
                return result;
            }

            void Validate(const TStringBuf&) override {
                if (Args.empty() || Args.size() > MAX_ARGS_COUNT) {
                    ythrow yexception() << "function 'fnvhash_f32' must have at least 1 and no more than 100 arguments. Has " << Args.size() << " arguments";
                }
            }

            static TFuncFactory::TRegistrator<TFactorCalcerHash> Registrator;
        };
        TFuncFactory::TRegistrator<TFactorCalcerHash> TFactorCalcerHash::Registrator("fnvhash_f32");

        class TFactorCalcerStringCityHash : public TFactorCalcerFunc {
            ui64 CalcInternal(TCalcFactorsContext& ctx) const {
                if (Args.size() == 2) {
                    return ::CityHash64WithSeed(Args[0]->CalcBlob(ctx), Args[1]->CalcInt(ctx));
                }
                return ::CityHash64(Args[0]->CalcBlob(ctx));
            }
        public:
            float DoCalc(TCalcFactorsContext& ctx) const override {
                double result = static_cast<double>(CalcInternal(ctx));
                result /= std::numeric_limits<ui64>::max();
                return result;
            }

            i64 DoCalcInt(TCalcFactorsContext& ctx) const override {
                return static_cast<i64>(CalcInternal(ctx));
            }

            void Validate(const TStringBuf&) override {
                if (Args.empty() || Args.size() > 2) {
                    ythrow yexception() << "function 'cityhash' must have either 1 (string) or 2 (string, int) arguments. Has " << Args.size() << " arguments";
                }
            }

            static TFuncFactory::TRegistrator<TFactorCalcerStringCityHash> Registrator;
        };
        TFuncFactory::TRegistrator<TFactorCalcerStringCityHash> TFactorCalcerStringCityHash::Registrator("cityhash");

        class TFactorCalcerRelevParamProvider : public TFactorCalcerFunc {
            template<typename T>
            inline T GetRelevParam(TCalcFactorsContext& ctx, T defaultValue = {}) const {
                Y_ENSURE(RP != nullptr);
                return RP->GetRelevParam<T>(Args[0]->CalcBlob(ctx), defaultValue);
            }
        public:
            float DoCalc(TCalcFactorsContext& ctx) const override {
                return GetRelevParam<float>(ctx);
            }

            i64 DoCalcInt(TCalcFactorsContext& ctx) const override {
                return GetRelevParam<i64>(ctx);
            }

            TString DoCalcBlob(TCalcFactorsContext& ctx) const override {
                return GetRelevParam<TString>(ctx);
            }

            void Validate(const TStringBuf&) override {
                if (Args.size() != 1) {
                    ythrow yexception() << "function 'get_relev' must have a single argument";
                }
            }

            TStringBuf GetCalculatorName() const override {
                return "get_relev";
            }

            static TFuncFactory::TRegistrator<TFactorCalcerRelevParamProvider> Registrator;
        };
        TFuncFactory::TRegistrator<TFactorCalcerRelevParamProvider> TFactorCalcerRelevParamProvider::Registrator("get_relev");

        // 0xc70f57c6 is a crc32 of TParsedModelEmbedding
        struct TParsedModelEmbedding: public StoredType<0xc70f57c6> {
            TVector<float> Value;
            TString Version;
        };

        // 0xbca204d4 is a crc32 of TDocDssmArrayWithTags
        struct TDocDssmArrayWithTags: public StoredType<0xbca204d4> {
            TVector<TVector<float>> Values;
            TVector<TString> Tags;
        };

        // 0x9b3266e8 is a crc32 of TFloatVector
        struct TFloatVector: public StoredType<0x9b3266e8> {
            TVector<float> Values;
        };

        // 0xbefb4412 is a crc32 of TDocEmbeddingsByVersion
        struct TDocEmbeddingsByVersion: public StoredType<0xbefb4412> {
            THashMap<TString, TVector<float>> Values;
        };

        class TSaveFloatVector: public TStoreValueFunc {
        public:
            const IStoredType* DoGet(TCalcFactorsContext& ctx, ui32 typeId) override {
                if (typeId != TFloatVector::TypeId) {
                    return nullptr;
                }
                if (Value.Empty()) {
                    Value = TFloatVector();
                    Value->Values = Args[0]->CalcFloatVector(ctx);
                }

                return Value.Get();
            }

            void Validate(const TStringBuf&) override {
                if (Args.size() != 1) {
                    ythrow yexception() << "function 'save_float_vector' must have a single blob argument";
                }
            }

            void Clear() override {
                Value = Nothing();
            }

            static TStoreValueFuncFactory::TRegistrator<TSaveFloatVector> Registrator;
        private:
            TMaybe<TFloatVector> Value;
        };
        TStoreValueFuncFactory::TRegistrator<TSaveFloatVector> TSaveFloatVector::Registrator("save_float_vector");

        class TLoadFloatVector: public TFactorCalcerFunc {
        public:
            TVector<float> DoCalcFloatVector(TCalcFactorsContext& ctx) const override {
                Y_ENSURE(ValuesStorage != nullptr);
                const TString& name = Args[0]->CalcBlob(ctx);
                const TFloatVector* parsed = ValuesStorage->Get<TFloatVector>(name, ctx);
                if (Y_UNLIKELY(parsed == nullptr)) {
                    ythrow yexception() << "no stored value for " << name;
                }
                if (ValuesStorage->DependsOnDoc(name)) {
                    SetDependsOnDoc();
                }
                return parsed->Values;
            }

            TStringBuf GetCalculatorName() const override {
                return "load_float_vector";
            }

            void Validate(const TStringBuf&) override {
                if (Args.size() != 1) {
                    ythrow yexception() << "function " << GetCalculatorName() << " must have a single string argument";
                }
            }

            static TFuncFactory::TRegistrator<TLoadFloatVector> Registrator;
        };
        TFuncFactory::TRegistrator<TLoadFloatVector> TLoadFloatVector::Registrator("load_float_vector");

        class TElementAt: public TFactorCalcerFunc {
        public:
            float DoCalc(TCalcFactorsContext& ctx) const override {
                Y_ENSURE(ValuesStorage != nullptr);
                const TVector<float>& values = Args[0]->CalcFloatVector(ctx);
                size_t index = Args[1]->CalcInt(ctx);
                if (index >= values.size()) {
                    return 0;
                }
                return values[index];
            }

            TStringBuf GetCalculatorName() const override {
                return "element_at";
            }

            void Validate(const TStringBuf&) override {
                if (Args.size() != 2) {
                    ythrow yexception() << "function " << GetCalculatorName() << " must have two arguments: floatVector, index";
                }
            }

            static TFuncFactory::TRegistrator<TElementAt> Registrator;
        };
        TFuncFactory::TRegistrator<TElementAt> TElementAt::Registrator("element_at");

        class TSaveDocDssmArrayWithTags: public TStoreValueFunc {
        public:
            const IStoredType* DoGet(TCalcFactorsContext& ctx, ui32 typeId) override {
                if (typeId != TDocDssmArrayWithTags::TypeId) {
                    return nullptr;
                }
                if (EmbeddingsWithTags.Defined()) {
                    return EmbeddingsWithTags.Get();
                }

                TDocDssmArrayWithTags result;
                result.Values = Args[0]->CalcFloatVectors(ctx);

                result.Tags = Args[0]->CalcBlobVector(ctx);

                EmbeddingsWithTags = std::move(result);
                return EmbeddingsWithTags.Get();
            }

            void Validate(const TStringBuf&) override {
                if (Args.size() != 1) {
                    ythrow yexception() << "function 'save_embeds_and_tags' must have a single blob argument";
                }
            }

            void Clear() override {
                EmbeddingsWithTags = Nothing();
            }

            static TStoreValueFuncFactory::TRegistrator<TSaveDocDssmArrayWithTags> Registrator;
        private:
            TMaybe<TDocDssmArrayWithTags> EmbeddingsWithTags;
        };
        TStoreValueFuncFactory::TRegistrator<TSaveDocDssmArrayWithTags> TSaveDocDssmArrayWithTags::Registrator("save_embeds_and_tags");

        class TFactorCalcerEmbedsAndTagsValues: public TFactorCalcerFunc {
        public:
            TVector<TVector<float>> DoCalcFloatVectors(TCalcFactorsContext& ctx) const override {
                Y_ENSURE(ValuesStorage != nullptr);
                const TString& name = Args[0]->CalcBlob(ctx);
                const TDocDssmArrayWithTags* parsed = ValuesStorage->Get<TDocDssmArrayWithTags>(name, ctx);
                if (Y_UNLIKELY(parsed == nullptr)) {
                    ythrow yexception() << "no stored value for " << name;
                }
                if (ValuesStorage->DependsOnDoc(name)) {
                    SetDependsOnDoc();
                }
                return parsed->Values;
            }

            TStringBuf GetCalculatorName() const override {
                return "load_embeds_values";
            }

            void Validate(const TStringBuf&) override {
                if (Args.size() != 1) {
                    ythrow yexception() << "function " << GetCalculatorName() << " must have a single string argument";
                }
            }

            static TFuncFactory::TRegistrator<TFactorCalcerEmbedsAndTagsValues> Registrator;
        };
        TFuncFactory::TRegistrator<TFactorCalcerEmbedsAndTagsValues> TFactorCalcerEmbedsAndTagsValues::Registrator("load_embeds_values");

        class TFactorCalcerEmbedsAndTagsTags: public TFactorCalcerFunc {
        public:
            TVector<TString> DoCalcBlobVector(TCalcFactorsContext& ctx) const override {
                Y_ENSURE(ValuesStorage != nullptr);
                const TString& name = Args[0]->CalcBlob(ctx);
                const TDocDssmArrayWithTags* parsed = ValuesStorage->Get<TDocDssmArrayWithTags>(name, ctx);
                if (Y_UNLIKELY(parsed == nullptr)) {
                    ythrow yexception() << "no stored value for " << name;
                }
                if (ValuesStorage->DependsOnDoc(name)) {
                    SetDependsOnDoc();
                }
                return parsed->Tags;
            }

            TStringBuf GetCalculatorName() const override {
                return "load_embeds_tags";
            }

            void Validate(const TStringBuf&) override {
                if (Args.size() != 1) {
                    ythrow yexception() << "function " << GetCalculatorName() << " must have a single string argument";
                }
            }

            static TFuncFactory::TRegistrator<TFactorCalcerEmbedsAndTagsTags> Registrator;
        };
        TFuncFactory::TRegistrator<TFactorCalcerEmbedsAndTagsTags> TFactorCalcerEmbedsAndTagsTags::Registrator("load_embeds_tags");

        class TPreloadDocEmbeddings: public TStoreValueFunc {
        public:
            const IStoredType* DoGet(TCalcFactorsContext& ctx, ui32 typeId) override {
                if (typeId != TDocEmbeddingsByVersion::TypeId) {
                    return nullptr;
                }
                if (EmbeddingsByVersion.Defined()) {
                    return EmbeddingsByVersion.Get();
                }
                const auto versions = Args[0]->CalcBlobVector(ctx);
                const auto embeddings = Args[0]->CalcFloatVectors(ctx);
                Y_ENSURE(versions.size() == embeddings.size(), "versions and embeddings size mismatch");

                TDocEmbeddingsByVersion result;
                for (size_t i = 0; i < versions.size(); i++) {
                    result.Values[std::move(versions[i])] = std::move(embeddings[i]);
                }
                EmbeddingsByVersion = std::move(result);
                return EmbeddingsByVersion.Get();
            }

            void Clear() override {
                EmbeddingsByVersion = Nothing();
            }

            static TStoreValueFuncFactory::TRegistrator<TPreloadDocEmbeddings> Registrator;
        private:
            TMaybe<TDocEmbeddingsByVersion> EmbeddingsByVersion;
        };
        TStoreValueFuncFactory::TRegistrator<TPreloadDocEmbeddings> TPreloadDocEmbeddings::Registrator("preload_doc_embeddings");

        class TFactorCalcerDocEmbeddingExists: public TFactorCalcerFunc {
        public:
            float DoCalc(TCalcFactorsContext& ctx) const override {
                Y_ENSURE(ValuesStorage);
                const TString& name = Args[0]->CalcBlob(ctx);
                const auto* cachedEmbeddings = ValuesStorage->Get<TDocEmbeddingsByVersion>(name, ctx);
                if (!cachedEmbeddings) {
                    ythrow yexception() << "no stored value for " << name;
                }

                const TString& version = Args[1]->CalcBlob(ctx);
                if (cachedEmbeddings->Values.contains(version)) {
                    return 1.0f;
                }
                return 0.0f;
            }

            TStringBuf GetCalculatorName() const override {
                return "doc_embedding_exists";
            }

            void Validate(const TStringBuf&) override {
                if (Args.size() != 2) {
                    ythrow yexception() << "function " << GetCalculatorName() << " must have two arguments";
                }
            }
            
            static TFuncFactory::TRegistrator<TFactorCalcerDocEmbeddingExists> Registrator;
        };
        TFuncFactory::TRegistrator<TFactorCalcerDocEmbeddingExists> TFactorCalcerDocEmbeddingExists::Registrator("doc_embedding_exists");

        class TFactorCalcerLoadCachedEmbedding: public TFactorCalcerFunc {
        public:
            TVector<float> DoCalcFloatVector(TCalcFactorsContext& ctx) const override {
                const TString& cacherName= Args[0]->CalcBlob(ctx);
                const auto* cachedEmbeddings = ValuesStorage->Get<TDocEmbeddingsByVersion>(cacherName, ctx);
                if (!cachedEmbeddings) {
                    ythrow yexception() << "no stored value for " << cacherName;
                }

                const TString& version = Args[1]->CalcBlob(ctx);
                auto it = cachedEmbeddings->Values.find(version);
                if (it == cachedEmbeddings->Values.end()) {
                    ythrow yexception() << "no cached embedding for (" << cacherName<< "," << version << ")";
                }
                return it->second;
            }

            TStringBuf GetCalculatorName() const override {
                return "load_cached_embedding";
            }

            void Validate(const TStringBuf&) override {
                if (Args.size() != 2) {
                    ythrow yexception() << "function " << GetCalculatorName() << " must have two arguments";
                }
            }

            static TFuncFactory::TRegistrator<TFactorCalcerLoadCachedEmbedding> Registrator;
        };
        TFuncFactory::TRegistrator<TFactorCalcerLoadCachedEmbedding> TFactorCalcerLoadCachedEmbedding::Registrator("load_cached_embedding");

        class TParseDssmModelEmbedding: public TStoreValueFunc {
        public:
            const IStoredType* DoGet(TCalcFactorsContext& ctx, ui32 typeId) override {
                if (typeId != TParsedModelEmbedding::TypeId) {
                    return nullptr;
                }
                if (ParsedEmbedding.Defined()) {
                    return ParsedEmbedding.Get();
                }

                const TString value = Args[0]->CalcBlob(ctx);

                NEmbeddingsTransfer::NProto::TModelEmbedding modelEmbedding;
                Y_ENSURE(modelEmbedding.ParseFromString(value), "failed to parse TModelEmbedding proto");

                TParsedModelEmbedding result;
                result.Version = modelEmbedding.GetVersion();
                result.Value = NEmbeddingsTransfer::DecompressEmbedding(
                    modelEmbedding.GetCompressedData(),
                    modelEmbedding.GetCompressionType()
                );
                NNeuralNetApplier::NormalizeVector(result.Value);
                ParsedEmbedding = std::move(result);
                return ParsedEmbedding.Get();
            }
            void Clear() override {
                ParsedEmbedding = Nothing();
            }

            void Validate(const TStringBuf&) override {
                if (Args.size() != 1) {
                    ythrow yexception() << "function 'parse_bg_model_embed' must have a single blob argument";
                }
            }

            static TStoreValueFuncFactory::TRegistrator<TParseDssmModelEmbedding> Registrator;
        private:
            TMaybe<TParsedModelEmbedding> ParsedEmbedding;
        };
        TStoreValueFuncFactory::TRegistrator<TParseDssmModelEmbedding> TParseDssmModelEmbedding::Registrator("parse_bg_model_embed");


        class TFactorCalcerModelEmbedVersion: public TFactorCalcerFunc {
        public:
            TString DoCalcBlob(TCalcFactorsContext& ctx) const override {
                Y_ENSURE(ValuesStorage != nullptr);
                const TString& name = Args[0]->CalcBlob(ctx);
                const TParsedModelEmbedding* parsed = ValuesStorage->Get<TParsedModelEmbedding>(name, ctx);
                if (Y_UNLIKELY(parsed == nullptr)) {
                    ythrow yexception() << "no stored value for " << name;
                }
                return parsed->Version;
            }

            TStringBuf GetCalculatorName() const override {
                return "bg_model_embed_version";
            }

            void Validate(const TStringBuf&) override {
                if (Args.size() != 1) {
                    ythrow yexception() << "function " << GetCalculatorName() << " must have a single string argument";
                }
            }

            static TFuncFactory::TRegistrator<TFactorCalcerModelEmbedVersion> Registrator;
        };
        TFuncFactory::TRegistrator<TFactorCalcerModelEmbedVersion> TFactorCalcerModelEmbedVersion::Registrator("bg_model_embed_version");


        class TFactorCalcerModelEmbedValue: public TFactorCalcerFunc {
        public:
            TVector<float> DoCalcFloatVector(TCalcFactorsContext& ctx) const override {
                Y_ENSURE(ValuesStorage != nullptr);
                const TString& name = Args[0]->CalcBlob(ctx);
                const TParsedModelEmbedding* parsed = ValuesStorage->Get<TParsedModelEmbedding>(name, ctx);
                if (Y_UNLIKELY(parsed == nullptr)) {
                    ythrow yexception() << "no stored value for " << name;
                }
                return parsed->Value;
            }

            TStringBuf GetCalculatorName() const override {
                return "bg_model_embed_value";
            }

            void Validate(const TStringBuf&) override {
                if (Args.size() != 1) {
                    ythrow yexception() << "function " << GetCalculatorName() << " must have a single string argument";
                }
            }

            static TFuncFactory::TRegistrator<TFactorCalcerModelEmbedValue> Registrator;
        };
        TFuncFactory::TRegistrator<TFactorCalcerModelEmbedValue> TFactorCalcerModelEmbedValue::Registrator("bg_model_embed_value");


        class TFactorCalcerDotProduct: public TFactorCalcerFunc {
        public:
            float DoCalc(TCalcFactorsContext& ctx) const override {
                const auto& vec1 = Args[0]->CalcFloatVector(ctx);
                const auto& vec2 = Args[1]->CalcFloatVector(ctx);
                if (vec1.size() != vec2.size()) {
                    ythrow yexception() << GetCalculatorName() << " arguments should have equal cardinality, " << vec1.size() << " != " << vec2.size();
                }

                return DotProduct(vec1.data(), vec2.data(), vec1.size());
            }

            void Validate(const TStringBuf&) override {
                if (Args.size() != 2) {
                    ythrow yexception() << "function '" << GetCalculatorName() << "' must have two arguments";
                }
            }

            TStringBuf GetCalculatorName() const override {
                return "dot_product";
            }

            static TFuncFactory::TRegistrator<TFactorCalcerDotProduct> Registrator;
        };
        TFuncFactory::TRegistrator<TFactorCalcerDotProduct> TFactorCalcerDotProduct::Registrator("dot_product");

        class TFactorCalcerVectorMaxElement: public TFactorCalcerFunc {
             float DoCalc(TCalcFactorsContext& ctx) const override {
                const auto& v = Args[0]->CalcFloatVector(ctx);
                if (v.empty()) {
                    if (Args.size() < 2) {
                        ythrow yexception() << GetCalculatorName() << " from an empty vector, and there is no defaultValue";
                    }
                    return Args[1]->Calc(ctx);
                }

                auto it = std::max_element(v.begin(), v.end());
                Y_ASSERT(it != v.end());
                return *it;
            }

            void Validate(const TStringBuf&) override {
                if (Args.empty() || Args.size() > 2) {
                    ythrow yexception() << "function '" << GetCalculatorName() << "' must have one or two arguments (vector, defaultValue[optional])";
                }
            }

            TStringBuf GetCalculatorName() const override {
                return "vect_max_element";
            }

            static TFuncFactory::TRegistrator<TFactorCalcerVectorMaxElement> Registrator;
        };
        TFuncFactory::TRegistrator<TFactorCalcerVectorMaxElement> TFactorCalcerVectorMaxElement::Registrator("vect_max_element");

        class TFactorCalcerVectorTopNElements: public TFactorCalcerFunc {
            TVector<float> DoCalcFloatVector(TCalcFactorsContext& ctx) const override {
                const auto& v = Args[0]->CalcFloatVector(ctx);
                size_t count = Args[1]->CalcInt(ctx);
                TVector<float> result(Reserve(count));

                if (Args.size() < 5) { // when there is no tags to process
                    if (v.size() < count) {
                        if (Args.size() < 3) {
                            ythrow yexception() << GetCalculatorName() << " vector size (" << v.size() << ") is less than desired count (" << count << "), and there is no defaultValue";
                        }
                        result.resize(count, Args[2]->Calc(ctx));
                    } else {
                        result.resize(count);
                    }
                    std::partial_sort_copy(v.begin(), v.end(), result.begin(), result.end(), std::greater<float>());
                    return result;
                }
                Y_ENSURE(Args.size() == 5, "there is insufficient amount of arguments");
                const auto& tags = Args[3]->CalcBlobVector(ctx);
                const auto& attrToStoreTag = Args[4]->CalcBlob(ctx);

                Y_ENSURE(v.size() == tags.size(), "vector of values and vector of tags must have equal cardinality");
                TVector<TString> tagsReordered;

                if (v.size() < count) {
                    const auto& defaultValue = Args[2]->Calc(ctx);
                    result.resize(count, defaultValue);
                    tagsReordered.resize(count, "");

                    auto tagsIndices = NTrafarets::NAlgo::ArgSort(v, std::greater<float>());
                    for (size_t i = 0; i < v.size(); i++) {
                        result[i] = v[tagsIndices[i]];
                        tagsReordered[i] = tags[tagsIndices[i]];
                    }
                    // need to check if default value is greater than some values
                } else {
                    result.resize(count);
                    tagsReordered.resize(count);
                    std::partial_sort_copy(v.begin(), v.end(), result.begin(), result.end(), std::greater<float>());
                    auto nth_max = result.back(); // complexity is n log(count)

                    TVector<float> topValues;
                    TVector<int> topIndices;
                    for (size_t i = 0; i < v.size(); i++) {
                        if (v[i] >= nth_max) {
                            topValues.push_back(v[i]);
                            topIndices.push_back(i);
                        }
                    }
                    auto tagsIndices = NTrafarets::NAlgo::ArgSort(topValues, std::greater<float>());
                    for (size_t i = 0; i < result.size(); i++) {
                        tagsReordered[i] = tags[topIndices[tagsIndices[i]]]; // hard to read
                    }
                }
                (*UserDefinedAttrs)[std::make_pair(ctx.DocId, attrToStoreTag)] = JoinSeq(";", tagsReordered);
                return result;
            }

            void Validate(const TStringBuf&) override {
                if (Args.size() < 2 || Args.size() > 5) {
                    ythrow yexception() << "function '" << GetCalculatorName() << "' must from two to five arguments (values, topSize, defaultValue[optional], tags[optional], fsgtaToStoreTags[optional])";
                }
            }

            TStringBuf GetCalculatorName() const override {
                return "vect_top_n_elements";
            }

            static TFuncFactory::TRegistrator<TFactorCalcerVectorTopNElements> Registrator;
        };
        TFuncFactory::TRegistrator<TFactorCalcerVectorTopNElements> TFactorCalcerVectorTopNElements::Registrator("vect_top_n_elements");

        class TFactorCalcerVectorMean: public TFactorCalcerFunc {
             float DoCalc(TCalcFactorsContext& ctx) const override {
                const auto& v = Args[0]->CalcFloatVector(ctx);
                if (v.empty()) {
                    if (Args.size() < 2) {
                        ythrow yexception() << GetCalculatorName() << " from an empty vector, and there is no defaultValue";
                    }
                    return Args[1]->Calc(ctx);
                }

                const float s = std::accumulate(v.begin(), v.end(), 0.0f);
                return s / v.size();
            }

            void Validate(const TStringBuf&) override {
                if (Args.empty() || Args.size() > 2) {
                    ythrow yexception() << "function '" << GetCalculatorName() << "' must have one or two arguments (vector, defaultValue[optional])";
                }
            }

            TStringBuf GetCalculatorName() const override {
                return "vect_mean";
            }

            static TFuncFactory::TRegistrator<TFactorCalcerVectorMean> Registrator;
        };
        TFuncFactory::TRegistrator<TFactorCalcerVectorMean> TFactorCalcerVectorMean::Registrator("vect_mean");

        class TFactorCalcerVectorHarmonicMean: public TFactorCalcerFunc {
             float DoCalc(TCalcFactorsContext& ctx) const override {
                const auto& v = Args[0]->CalcFloatVector(ctx);
                if (v.empty()) {
                    return GetDefault(ctx, "from an empty vector");
                }

                float s = 0.0f;
                int cnt = 0;
                for (const float f: v) {
                    if (f == 0.0f) {
                        continue;
                    }
                    s += (1.0f / f);
                    cnt++;
                }

                if (s == 0.0f) {
                    return GetDefault(ctx, "from a zero sum");
                }
                return cnt / s;
            }

            float GetDefault(TCalcFactorsContext& ctx, TStringBuf reason) const {
                if (Args.size() < 2) {
                    ythrow yexception() << GetCalculatorName() << " " << reason << ", and there is no defaultValue";
                }
                return Args[1]->Calc(ctx);
            }

            void Validate(const TStringBuf&) override {
                if (Args.empty() || Args.size() > 2) {
                    ythrow yexception() << "function '" << GetCalculatorName() << "' must have one or two arguments (vector, defaultValue[optional])";
                }
            }

            TStringBuf GetCalculatorName() const override {
                return "vect_harmonic_mean";
            }

            static TFuncFactory::TRegistrator<TFactorCalcerVectorHarmonicMean> Registrator;
        };
        TFuncFactory::TRegistrator<TFactorCalcerVectorHarmonicMean> TFactorCalcerVectorHarmonicMean::Registrator("vect_harmonic_mean");

        class TFactorCalcerMatrixOnVectorMultiplication: public TFactorCalcerFunc {
            TVector<float> DoCalcFloatVector(TCalcFactorsContext& ctx) const override {
                const auto& matr = Args[0]->CalcFloatVectors(ctx);
                const auto& vec = Args[1]->CalcFloatVector(ctx);

                TVector<float> result(matr.size());
                for (size_t rowIdx = 0; rowIdx < matr.size(); rowIdx++) {
                    const auto& row = matr[rowIdx];
                    if (row.size() != vec.size()) {
                        ythrow yexception() << GetCalculatorName() << " matrix rows should and vector should have equal cardinalities";
                    }
                    result[rowIdx] = DotProduct(row.data(), vec.data(), vec.size());
                }
                return result;
            }

            void Validate(const TStringBuf&) override {
                if (Args.size() != 2) {
                    ythrow yexception() << "function '" << GetCalculatorName() << "' must have two arguments (matrix, vector)";
                }
            }

            TStringBuf GetCalculatorName() const override {
                return "matr_X_vect";
            }

            static TFuncFactory::TRegistrator<TFactorCalcerMatrixOnVectorMultiplication> Registrator;
        };
        TFuncFactory::TRegistrator<TFactorCalcerMatrixOnVectorMultiplication> TFactorCalcerMatrixOnVectorMultiplication::Registrator("matr_X_vect");

        class TFactorCalcerBase64Decoder: public TFactorCalcerFunc {
        public:
            TString DoCalcBlob(TCalcFactorsContext& ctx) const override {
                return ::Base64Decode(Args[0]->CalcBlob(ctx));
            }

            void Validate(const TStringBuf&) override {
                if (Args.size() != 1) {
                    ythrow yexception() << "function 'base64_decode' must have a single binary argument";
                }
            }

            TStringBuf GetCalculatorName() const override {
                return "base64_decode";
            }

            static TFuncFactory::TRegistrator<TFactorCalcerBase64Decoder> Registrator;
        };
        TFuncFactory::TRegistrator<TFactorCalcerBase64Decoder> TFactorCalcerBase64Decoder::Registrator("base64_decode");

        class TFactorCalcerDssmDecoder : public TFactorCalcerFunc {
        public:
            TVector<float> DoCalcFloatVector(TCalcFactorsContext& ctx) const override {
                const TString& compressionAlgo = Args[1]->CalcBlob(ctx);
                if (compressionAlgo == "AutoMaxCoordRenorm") {
                    const auto& compressed = Args[0]->CalcBlob(ctx);
                    TVector<float> result = NDssmApplier::NUtils::TFloat2UI8Compressor::Decompress(
                        TConstArrayRef<ui8>{reinterpret_cast<const ui8*>(compressed.data()), compressed.size()}
                    );
                    NNeuralNetApplier::NormalizeVector(result);
                    return result;
                } else if (compressionAlgo == "Float32") {
                    const auto& compressed = Args[0]->CalcBlob(ctx);
                    TVector<float> result(compressed.size() / sizeof(float));
                    memcpy((void* )result.data(), compressed.data(), compressed.size());
                    return result;
                }


                const NNeuralNetApplier::EDssmModel modelType = ::FromString<NNeuralNetApplier::EDssmModel>(compressionAlgo);

                NNeuralNetApplier::IBegemotDssmDataPtr serializer = NNeuralNetApplier::GetDssmDataSerializer(static_cast<NNeuralNetApplier::EDssmModel>(modelType));
                TStringStream argStream(Args[0]->CalcBlob(ctx));
                serializer->Load(&argStream);

                TVector<float> result;
                serializer->TryGetLatestEmbedding(result);
                NNeuralNetApplier::NormalizeVector(result);
                return result;
            }

            void Validate(const TStringBuf&) override {
                if (Args.size() != 2) {
                    ythrow yexception() << "function 'dssm_decode' must have two arguments (what_to_decode, dssm_model)";
                }
            }

            TStringBuf GetCalculatorName() const override {
                return "dssm_decode";
            }

            static TFuncFactory::TRegistrator<TFactorCalcerDssmDecoder> Registrator;
        };
        TFuncFactory::TRegistrator<TFactorCalcerDssmDecoder> TFactorCalcerDssmDecoder::Registrator("dssm_decode");

        class TFactorCalcerDocIdHash : public TFactorCalcerFunc {
        public:
            i64 DoCalcInt(TCalcFactorsContext& ctx) const override {
                Y_ENSURE(DDKManager);
                return DDKManager->GetIdentifier(ctx.DocId).Head<i64>();
            }

            void Validate(const TStringBuf&) override {
                if (!Args.empty()) {
                    ythrow yexception() << "function 'zdocid_i64' must have no arguments";
                }
            }

            static TFuncFactory::TRegistrator<TFactorCalcerDocIdHash> Registrator;
        };
        TFuncFactory::TRegistrator<TFactorCalcerDocIdHash> TFactorCalcerDocIdHash::Registrator("zdocid_i64");

        class TFactorCalcerHypot : public TFactorCalcerFunc {
        public:
            Y_FORCE_INLINE static constexpr float Sqr(float vl) {
                return vl * vl;
            }

            float DoCalc(TCalcFactorsContext& ctx) const override {
                float sq = Sqr(Args[0]->Calc(ctx) - Args[2]->Calc(ctx)) + Sqr(Args[1]->Calc(ctx) - Args[3]->Calc(ctx));
                return sq > 0.f ? std::sqrtf(sq) : 0.f;
            }

            void Validate(const TStringBuf&) override {
                if (Args.size() != 4)
                    ythrow yexception() << "function 'hypot' must have exactly 4 arguments";
            }

            static TFuncFactory::TRegistrator<TFactorCalcerHypot> Registrator;
        };

        TFuncFactory::TRegistrator<TFactorCalcerHypot> TFactorCalcerHypot::Registrator("hypot");

        class TFactorCalcerClamp : public TFactorCalcerFunc {
        public:
            float DoCalc(TCalcFactorsContext& ctx) const override {
                // returns a value in [0..1] range
                float vl = Args[0]->Calc(ctx);
                if (Args.size() == 1) {
                    return std::clamp(vl, 0.f, 1.f);
                } else {
                    float zero = Args[1]->Calc(ctx);
                    float one = Args[2]->Calc(ctx);
                    if (one <= zero)
                        return 0.f;
                    return (std::clamp(vl, zero, one) - zero) / (one - zero);
                }
            }

            void Validate(const TStringBuf&) override {
                if (Args.size() != 1 && Args.size() != 3)
                    ythrow yexception() << "function 'clamp' must have 1 or 3 arguments";
            }

            static TFuncFactory::TRegistrator<TFactorCalcerClamp> Registrator;
        };

        TFuncFactory::TRegistrator<TFactorCalcerClamp> TFactorCalcerClamp::Registrator("clamp");

        class TFactorCalcerSigma : public TFactorCalcerFunc {
        public:
            float DoCalc(TCalcFactorsContext& ctx) const override {
                // returns a value in [0..1] range
                const float vl = Args[0]->Calc(ctx);
                const float mid = Args[1]->Calc(ctx); // the value that maps to 0.5
                const float q90 = Args[2]->Calc(ctx); // the value that maps to 0.9
                // Note: the Sigmoid is symmetric, hence the value that maps to 0.1 is (mid - (q90 - mid)).
                if (q90 == mid) {
                    return vl > mid ? 1.f : 0.f;
                }
                constexpr float ln10 = 2.302585092f;
                return Sigmoid(ln10 * (vl - mid) / (q90 - mid));
            }

            void Validate(const TStringBuf&) override {
                if (Args.size() != 3)
                    ythrow yexception() << "function 'sigma' must have exactly 3 arguments";
            }

            static TFuncFactory::TRegistrator<TFactorCalcerSigma> Registrator;
        };

        TFuncFactory::TRegistrator<TFactorCalcerSigma> TFactorCalcerSigma::Registrator("sigma");

        class TFactorCalcerOr : public TFactorCalcerFunc {
        public:
            float DoCalc(TCalcFactorsContext& ctx) const override {
                for (TArgs::const_iterator i = Args.begin(); i != Args.end(); ++i)
                    if (fabs((*i)->Calc(ctx)) > 1e-6)
                        return 1.f;
                return 0.f;
            }

            static TFuncFactory::TRegistrator<TFactorCalcerOr> Registrator;
        };

        TFuncFactory::TRegistrator<TFactorCalcerOr> TFactorCalcerOr::Registrator("or");

        class TFactorCalcerAnd : public TFactorCalcerFunc {
        public:
            float DoCalc(TCalcFactorsContext& ctx) const override {
                for (TArgs::const_iterator i = Args.begin(); i != Args.end(); ++i)
                    if (fabs((*i)->Calc(ctx)) <= 1e-6)
                        return 0.f;
                return 1.f;
            }

            static TFuncFactory::TRegistrator<TFactorCalcerAnd> Registrator;
        };

        TFuncFactory::TRegistrator<TFactorCalcerAnd> TFactorCalcerAnd::Registrator("and");

        class TFactorCalcerOneArg : public TFactorCalcerFunc {
        public:
            using TUserFactorsCalcer::IFactorCalcer::Init;

            void Validate(const TStringBuf& name) override {
                if (Args.size() != 1)
                    ythrow yexception() << "function '" << name << "' must have exactly 1 argument";
            }
        };

        class TFactorCalcerNo : public TFactorCalcerOneArg {
        public:
            float DoCalc(TCalcFactorsContext& ctx) const override {
                return (fabs(Args[0]->Calc(ctx)) < 1e-6f) ? 1.f : 0.f;
            }

            static TFuncFactory::TRegistrator<TFactorCalcerNo> Registrator;
        };

        TFuncFactory::TRegistrator<TFactorCalcerNo> TFactorCalcerNo::Registrator("no");

        class TFactorCalcerAbs : public TFactorCalcerOneArg {
        public:
            float DoCalc(TCalcFactorsContext& ctx) const override {
                return fabs(Args[0]->Calc(ctx));
            }

            static TFuncFactory::TRegistrator<TFactorCalcerAbs> Registrator;
        };

        TFuncFactory::TRegistrator<TFactorCalcerAbs> TFactorCalcerAbs::Registrator("abs");

        class TFactorCalcerLn : public TFactorCalcerOneArg {
        public:
            float DoCalc(TCalcFactorsContext& ctx) const override {
                return logf(Args[0]->Calc(ctx));
            }

            static TFuncFactory::TRegistrator<TFactorCalcerLn> Registrator;
        };

        TFuncFactory::TRegistrator<TFactorCalcerLn> TFactorCalcerLn::Registrator("ln");

        class TFactorCalcerLog10 : public TFactorCalcerOneArg {
        public:
            float DoCalc(TCalcFactorsContext& ctx) const override {
                return log10f(Args[0]->Calc(ctx));
            }

            static TFuncFactory::TRegistrator<TFactorCalcerLog10> Registrator;
        };

        TFuncFactory::TRegistrator<TFactorCalcerLog10> TFactorCalcerLog10::Registrator("log10");

        class TFactorCalcerMax : public TFactorCalcerFunc {
        public:
            float DoCalc(TCalcFactorsContext& ctx) const override {
                float result = Min<float>();
                for (TArgs::const_iterator i = Args.begin(); i != Args.end(); ++i)
                    result = Max<float>(result, (*i)->Calc(ctx));
                return result;
            }

            static TFuncFactory::TRegistrator<TFactorCalcerMax> Registrator;
        };

        TFuncFactory::TRegistrator<TFactorCalcerMax> TFactorCalcerMax::Registrator("max");

        class TFactorCalcerMin : public TFactorCalcerFunc {
        public:
            float DoCalc(TCalcFactorsContext& ctx) const override {
                float result = Max<float>();
                for (TArgs::const_iterator i = Args.begin(); i != Args.end(); ++i)
                    result = Min<float>(result, (*i)->Calc(ctx));
                return result;
            }

            static TFuncFactory::TRegistrator<TFactorCalcerMin> Registrator;
        };

        TFuncFactory::TRegistrator<TFactorCalcerMin> TFactorCalcerMin::Registrator("min");

        class TFactorCalcerSum : public TFactorCalcerFunc {
        public:
            float DoCalc(TCalcFactorsContext& ctx) const override {
                float result = 0;
                for (TArgs::const_iterator i = Args.begin(); i != Args.end(); ++i)
                    result += (*i)->Calc(ctx);
                return result;
            }

            static TFuncFactory::TRegistrator<TFactorCalcerSum> Registrator;
        };

        TFuncFactory::TRegistrator<TFactorCalcerSum> TFactorCalcerSum::Registrator("sum");

        class TFactorCalcerMul : public TFactorCalcerFunc {
        public:
            float DoCalc(TCalcFactorsContext& ctx) const override {
                float result = 1;
                for (TArgs::const_iterator i = Args.begin(); i != Args.end(); ++i)
                    result *= (*i)->Calc(ctx);
                return result;
            }

            static TFuncFactory::TRegistrator<TFactorCalcerMul> Registrator;
        };

        TFuncFactory::TRegistrator<TFactorCalcerMul> TFactorCalcerMul::Registrator("mul");

        class TFactorCalcerCoalesceZero: public TFactorCalcerFunc {
        public:
            float DoCalc(TCalcFactorsContext& ctx) const override {
                for (const auto& arg: Args) {
                    float result = arg->Calc(ctx);
                    if (fabs(result) > 1e-6) {
                        return result;
                    }
                }
                return 0.0;
            }

            void Validate(const TStringBuf&) override {
                if (Args.empty()) {
                    ythrow yexception() << "function '" << GetCalculatorName() << "' must have at least one argument";
                }
            }

            TStringBuf GetCalculatorName() const override {
                return "coalesce_zero";
            }

            static TFuncFactory::TRegistrator<TFactorCalcerCoalesceZero> Registrator;
        };
        TFuncFactory::TRegistrator<TFactorCalcerCoalesceZero> TFactorCalcerCoalesceZero::Registrator("coalesce_zero");

        class TFactorCalcerImportedFunc : public TFactorCalcerFunc {
        private:
            NRTYFeatures::TFactorCalcerUserFuncEx Import_;

        public:
            // TFactorCalcerImportedFunc is not accessible through TFuncFactory, so no default constructor
            TFactorCalcerImportedFunc(const NRTYFeatures::TFactorCalcerUserFuncEx& importedFunc)
                : Import_(importedFunc)
            {
                Y_ENSURE(importedFunc);
                SetDependsOnDoc();
            }

            float DoCalc(TCalcFactorsContext& ctx) const override {
                TVector<float> argValues(Args.size());
                auto o = argValues.begin();
                for (TArgs::const_iterator i = Args.begin(); i != Args.end(); ++i)
                    *o++ = (*i)->Calc(ctx);

                const TRTYFunctionCtx& uctx = Default<TRTYFunctionCtx>(); //TODO: provide uctx.SearchStatistics
                return Import_(ctx.DocId, TArrayRef<const float>(argValues.cbegin(), argValues.cend()), uctx);
            }

            void Validate(const TStringBuf& name) override {
                if (Args.size() < Import_.MinArgsCount || Args.size() > Import_.MaxArgsCount) {
                    ythrow yexception() << "function '" << name << "' must have from "
                        << Import_.MinArgsCount << " to " << Import_.MaxArgsCount << " arguments";
                }
            }
        };

        class TFactorCalcerImportedGenericFunc : public TFactorCalcerFunc {
        private:
            NRTYFeatures::TFactorCalcerGenericUserFunc Import_;

            THolder<NRTYFactors::IFactorCalculate> GetCalcer(TCalcFactorsContext& ctx) const {
                Y_ENSURE(SearchStatistics);
                TVector<const NRTYFactors::IFactorCalculate*> argValues(Args.size());
                auto o = argValues.begin();
                for (TArgs::const_iterator i = Args.begin(); i != Args.end(); ++i) {
                    *o++ = static_cast<NRTYFactors::IFactorCalculate*>(i->Get());
                }
                TRTYFunctionCtx uctx;
                uctx.SearchStatistics = SearchStatistics;
                THolder<NRTYFactors::IFactorCalculate> result = Import_(ctx, uctx, TArrayRef<const NRTYFactors::IFactorCalculate*>(argValues.begin(), argValues.end()));
                Y_ENSURE(result);
                return result;
            }

        public:
            TFactorCalcerImportedGenericFunc(const NRTYFeatures::TFactorCalcerGenericUserFunc& importedFunc)
                : Import_(importedFunc)
            {
                Y_ENSURE(importedFunc);
                SetDependsOnDoc();
            }

            float DoCalc(TCalcFactorsContext& ctx) const override {
                return GetCalcer(ctx)->Calc(ctx);
            }

            i64 DoCalcInt(TCalcFactorsContext& ctx) const override {
                return GetCalcer(ctx)->CalcInt(ctx);
            }

            TString DoCalcBlob(TCalcFactorsContext& ctx) const override {
                return GetCalcer(ctx)->CalcBlob(ctx);
            }

            TVector<float> DoCalcFloatVector(TCalcFactorsContext& ctx) const override {
                return GetCalcer(ctx)->CalcFloatVector(ctx);
            }

            TVector<TVector<float> > DoCalcFloatVectors(TCalcFactorsContext& ctx) const override {
                return GetCalcer(ctx)->CalcFloatVectors(ctx);
            }

            TVector<TString> DoCalcBlobVector(TCalcFactorsContext& ctx) const override {
                return GetCalcer(ctx)->CalcBlobVector(ctx);
            }

            void Validate(const TStringBuf& name) override {
                if (Args.size() < Import_.MinArgsCount || Args.size() > Import_.MaxArgsCount) {
                    ythrow yexception() << "function '" << name << "' must have from "
                        << Import_.MinArgsCount << " to " << Import_.MaxArgsCount << " arguments";
                }
            }
        };

        class TFactorCalcerImportedLayeredFunc : public TFactorCalcerFunc {
        private:
            NRTYFeatures::TFactorCalcerLayeredFunc Import_;
            bool HasSpecialArg;
        public:
            // TFactorCalcerImportedLayeredFunc is not accessible through TFuncFactory, so no default constructor
            TFactorCalcerImportedLayeredFunc(const NRTYFeatures::TFactorCalcerLayeredFunc& importedFunc)
                : Import_(importedFunc)
            {
                Y_ENSURE(importedFunc);
                SetDependsOnDoc();
            }

            float DoCalc(TCalcFactorsContext& ctx) const override {
                i64 layerId;
                TArgs::const_iterator i = Args.begin();
                size_t passedArgsCnt = Args.size();
                if (HasSpecialArg) {
                    layerId = (*i++)->CalcInt(ctx);
                    passedArgsCnt--;
                } else {
                    layerId = 0;
                }

                TVector<float> argValues(passedArgsCnt);
                auto o = argValues.begin();
                for (; i != Args.end(); ++i) {
                    *o++ = (*i)->Calc(ctx);
                }
                const TRTYFunctionCtx& uctx = Default<TRTYFunctionCtx>(); //TODO: provide uctx.SearchStatistics
                return Import_(ctx.DocId, layerId, TArrayRef<const float>(argValues.cbegin(), argValues.cend()), uctx);
            }

            using TFactorCalcerFunc::Init;

            void Init(const TStringBuf& src, const TStringBuf& name, size_t& pos, const TInitCalcerParams& params) override {
                TFactorCalcerFunc::Init(src, name, pos, params);
                HasSpecialArg = !Args.empty() && !Args[0]->DescribeAs(EExprType::SecondaryKey).empty();
            }

            void Validate(const TStringBuf& name) override {
                const ui32 leftArgs = Args.size() - (HasSpecialArg ? 1 : 0);
                if (leftArgs < Import_.MinArgsCount || leftArgs > Import_.MaxArgsCount) {
                    ythrow yexception() << "function '" << name << "' must have from "
                        << Import_.MinArgsCount << " to " << Import_.MaxArgsCount << " arguments";
                }
            }
        };

        [[nodiscard]]
        TFactorCalcerFunc* TryConstructImportedFunc(const TStringBuf& funcName, const TImportedFunctions::TData& registry) {
            // lookup (by funcName) a custom imported function
            auto simple = registry.UserFuncs.find(funcName);
            if (simple != registry.UserFuncs.end()) {
                return new TFactorCalcerImportedFunc(simple->second);
            }
            auto generic = registry.GenericUserFuncs.find(funcName);
            if (generic != registry.GenericUserFuncs.end()) {
                return new TFactorCalcerImportedGenericFunc(generic->second);
            }
            auto layered = registry.LayeredFuncs.find(funcName);
            if (layered != registry.LayeredFuncs.end()) {
                return new TFactorCalcerImportedLayeredFunc(layered->second);
            }
            return nullptr;
        }

        TUserFactorsCalcer::IFactorCalcer* CreateFuncFromString(const TStringBuf& src, size_t& pos, TInitCalcerParams params) {
            const size_t endname = src.find_first_of(" ,)(", pos);

            const TStringBuf funcName(src.substr(pos, endname - pos));

            TFactorCalcerFunc* func = TFuncFactory::Construct(funcName);
            if (!func && params.Imports) {
                func = TryConstructImportedFunc(funcName, params.Imports->GetFunctions());
            }
            if (!func) {
                ythrow yexception() << "Unknown function " << funcName;
            }

            pos = endname + 1;
            func->Init(src, funcName, pos, params);
            func->Validate(funcName);
            return func;
        }

        TUserFactorsCalcer::IFactorCalcer* CreateCalcerFromString(const TStringBuf& src, size_t& pos, const TInitCalcerParams& params) {
            if (!SkipSpaces(src, pos))
                return nullptr;

            const char c = src[pos];
            if (c >= '0' && c <= '9' || c == '-' || c == '.' || c == '"') {
                return new TFactorCalcerConst(src, pos);
            }

            if (c == '#') {
                auto tail = src.substr(pos);
                if (tail.StartsWith(TFactorCalcerGroupAttr::Prefix)) {
                    if (!params.DocsAttrs)
                        ythrow yexception() << "there is no grouping attrs";
                    return new TFactorCalcerGroupAttr(src, pos, *params.DocsAttrs);
                }
                if (tail.StartsWith(TFactorCalcerGeoLayer::Prefix)) {
                    return new TFactorCalcerGeoLayer(src, pos, params.FactorConfig);
                }
                return new TFactorCalcerFactor(src, pos, params.FactorConfig);
            }
            return CreateFuncFromString(src, pos, params);
        }

        THolder<IStoreValueCalcer> CreateValueStoreFromString(const TStringBuf& src, size_t& pos, TInitCalcerParams& params) {
            if (!SkipSpaces(src, pos)) {
                return {};
            }

            const size_t endname = src.find_first_of(" ,)(", pos);
            const TStringBuf funcName(src.substr(pos, endname - pos));

            TStoreValueFunc* func = TStoreValueFuncFactory::Construct(funcName);
            if (!func) {
                ythrow yexception() << "Unknown store function " << funcName;
            }

            pos = endname + 1;
            func->Init(src, funcName, pos, params);
            func->Validate(funcName);
            return THolder(func);
        }
    } // namespace NCalcers

    //
    // TFormulaUserFactorsCalcer
    //
    TUserFactorsCalcer::TFormulaUserFactorsCalcer::TFormulaUserFactorsCalcer(const TUserFactorsCalcer& owner, const TSet<ui32>& usedFactors)
        : CommonCalcers(owner.Calcers)
        , SearchStatistics(owner.SearchStatistics)
        , UserDefinedAttrs(owner.GetUserDefinedAttrs()) {
        TMap<ui32, ui32> srcToDest;
        for (TSet<ui32>::const_iterator i = usedFactors.begin(); i != usedFactors.end(); ++i) {
            SetOrder(*i);
            TSetDstToSrc::const_iterator dst = owner.SetDstToSrc.find(*i);
            if (dst != owner.SetDstToSrc.end()) {
                srcToDest[dst->second] = dst->first;
                UsedFactors.insert(dst->second);
            }
        }
        for (TUserFactorsCalcer::TCalcers::const_iterator i = CommonCalcers.begin(); i != CommonCalcers.end(); ++i)
            if (i->second.GetOrder()) {
                Calcers.push_back(i->second);
                UsedFactors.insert(i->first);
                i->second.FillUsedFactors(UsedFactors);
            }

        for (TSet<ui32>::const_iterator i = UsedFactors.begin(); i != UsedFactors.end(); ++i) {
            TSetDstToSrc::const_iterator dst = owner.SetDstToSrc.find(*i);
            if (dst != owner.SetDstToSrc.end()) {
                srcToDest[dst->second] = dst->first;
                UsedFactors.insert(dst->second);
            }
        }
        TConfig::BuildChunks(CopyFactorsChunks, srcToDest);
        Sort(Calcers.begin(), Calcers.end());
    }

    ui32 TUserFactorsCalcer::TFormulaUserFactorsCalcer::SetOrder(ui32 index) {
        TUserFactorsCalcer::TCalcers::iterator i = CommonCalcers.find(index);
        if (i == CommonCalcers.end())
            return 0;
        ui32 max = 0;
        TIndexSet usedFactors;
        i->second.FillUsedFactors(usedFactors);
        for (TIndexSet::const_iterator ind = usedFactors.begin(); ind != usedFactors.end(); ++ind)
            max = Max<ui32>(max, SetOrder(*ind));
        i->second.SetOrder(++max);
        return max;
    }

    //
    // TUserFactorsCalcer
    //
    TUserFactorsCalcer::TUserFactorsCalcer(
        const TCalcers& calcers,
        const TMap<ui32, ui32>& setSrcToDst,
        const TConfig& config,
        THolder<TUserFactorsCalcer::TCachers>&& cachers,
        TRTYSearchStatistics& searchStatistics)
        : Cachers(std::move(cachers))
        , Calcers(calcers)
        , Config(config)
        , SearchStatistics(searchStatistics)
    {
        for (TMap<ui32, ui32>::const_iterator i = setSrcToDst.begin(); i != setSrcToDst.end(); ++i) {
            if (Calcers.find(i->second) != Calcers.end() || !SetDstToSrc.insert(std::make_pair(i->second, i->first)).second)
                ythrow yexception() << "factor " << Config.GetFactorName(i->second) << " set twice";
        }

        for (TCalcers::iterator i = Calcers.begin(); i != Calcers.end(); ++i)
            i->second.GetCalcer()->Init(Calcers);

        for (TCalcers::iterator cl = Calcers.begin(); cl != Calcers.end(); ++cl) {
            TString errors;
            if (!CheckCircles(cl->second, errors))
                ythrow yexception() << "circle on calculating user factors: " << errors;
        }

        for (TCalcers::iterator i = Calcers.begin(); i != Calcers.end(); ++i)
            i->second.SetOrder(0);
    }

    TUserFactorsCalcer::TFormulaUserFactorsCalcer* TUserFactorsCalcer::CreateFormulaCalcer(const TSet<ui32>& usedFactors) const {
        THolder<TFormulaUserFactorsCalcer> result(new TFormulaUserFactorsCalcer(*this, usedFactors));
        if (!result->IsEmpty())
            return result.Release();
        return nullptr;
    }

    bool TUserFactorsCalcer::CheckCircles(TFactorCalcerHolder& calcer, TString& errors) {
        if (calcer.GetOrder() == 1) {
            errors = Config.GetFactorName(calcer.GetIndex());
            return false;
        } else if (calcer.GetOrder() == 2)
            return true;
        calcer.SetOrder(1);
        TIndexSet usedFactors;
        calcer.FillUsedFactors(usedFactors);
        for (TIndexSet::const_iterator i = usedFactors.begin(); i != usedFactors.end(); ++i) {
            TCalcers::iterator cl = Calcers.find(*i);
            if (cl != Calcers.end()) {
                if (!CheckCircles(cl->second, errors)) {
                    errors = Config.GetFactorName(calcer.GetIndex()) + ("-" + errors);
                    return false;
                }
            } else if (Config.GetFactorByFormulaIndex(*i)->FactorType == ftUser && SetDstToSrc.find(*i) == SetDstToSrc.end())
                ythrow yexception() << "factor " << Config.GetFactorName(*i) << " not defined";
        }
        calcer.SetOrder(2);
        return true;
    }

    const TUserDefinedAttrsStorage* TUserFactorsCalcer::GetUserDefinedAttrs() const {
        Y_ENSURE(Cachers, "Cachers are uninitialized");
        return Cachers->AttrsStorage.Get();
    }

    TUserFactorsCalcer* TUserFactorsCalcer::Create(
            const TVector<TString>& calcDirectives, const TVector<TString>& storeDirectives, const TFactorPoliteness& reqParams,
            const TMap<ui32, ui32>& setSrcToDst, const TConfig& factorConfig, const NGroupingAttrs::TDocsAttrs* docsAttrs,
            const TImportedFunctions* imports, const IDDKManager* ddkManager, const TRequestParams* rp, TRTYSearchStatistics& searchStatistics,
            bool enableCacheInvalidation) {
        TRY {
            THolder<TUserFactorsCalcer::TCachers> cachers = MakeHolder<TUserFactorsCalcer::TCachers>(enableCacheInvalidation);
            NCalcers::TInitCalcerParams params{factorConfig, docsAttrs, imports, ddkManager, rp, searchStatistics, cachers->ValuesStorage.Get(), cachers->AttrsStorage.Get()};

            TCalcers calcers;
            for (TVector<TString>::const_iterator dir = calcDirectives.begin(); dir != calcDirectives.end(); ++dir) {
                size_t pos = dir->find(':');
                if (Y_UNLIKELY(pos == TString::npos))
                    ythrow yexception() << "invalid calc directive " << *dir;

                const TStringBuf factorName {dir->data(), pos};
                const TFactor* factorInfo = factorConfig.GetFactorByName(factorName);
                if (Y_UNLIKELY(factorInfo == nullptr)) {
                    if (!reqParams.AllowMissingUserFactors()) {
                        ythrow yexception() << "Unknown user factor " << factorName;
                    } else {
                        SEARCH_DEBUG << "Unknown user factor " << factorName << Endl;
                        continue;
                    }
                }
                const size_t dst = factorInfo->IndexGlobal;
                ++pos;

                TUserFactorsCalcer::IFactorCalcer* calcer = NCalcers::CreateCalcerFromString(TStringBuf(dir->data()), pos, params);
                if (calcer) {
                    if (!reqParams.AllowDoubleCalc() && calcers.find(dst) != calcers.end()) {
                        ythrow yexception() << "double calc for user factor " << *dir;
                    }
                    calcers[dst] = TFactorCalcerHolder(calcer, dst, factorInfo->DefaultValue, cachers->ValuesStorage, cachers->AttrsStorage, searchStatistics);
                }
            }
            if (calcers.empty() && setSrcToDst.empty())
                return  nullptr;

            for (const auto& dir: storeDirectives) {
                size_t pos = dir.find(':');
                if (Y_UNLIKELY(pos == TString::npos)) {
                    ythrow yexception() << "invalid store directive " << dir;
                }
                const TStringBuf variableName{dir.data(), pos};
                ++pos;

                THolder<IStoreValueCalcer> calcer = NCalcers::CreateValueStoreFromString(dir, pos, params);
                if (calcer) {
                    cachers->ValuesStorage->AddStoreValueCalcer(variableName, std::move(calcer), reqParams.AllowDoubleCalc());
                }
            }
            return new TUserFactorsCalcer(calcers, setSrcToDst, factorConfig, std::move(cachers), searchStatistics);
        } CATCH_AND_RETHROW("in TUserFactorsCalcer::Create")
        return nullptr;
    }
}
