#include "searcher_config.h"

#include "config.h"
#include "const.h"
#include "get_value.h"
#include "realm_config.h"
#include "shards_config.h"

#include <saas/rtyserver/factors/factors_config.h>
#include <saas/rtyserver/logging/rty_access.h>
#include <saas/rtyserver/pruning_config/pruning_config.h>

#include <search/config/virthost.h>
#include <search/grouping/groupinfo.h>

#include <kernel/qtree/request/reqattrlist.h>

#include <library/cpp/mediator/global_notifications/system_status.h>

#include <util/folder/dirut.h>
#include <util/string/join.h>
#include <util/string/vector.h>

namespace NRTYServer {

TSearcherConfig::TSearcherConfig(TRTYServerConfig& owner)
    : AutoStartServer(true)
    , DelegateRequestOptimization(false)
    , Enabled(true)
    , EnableUrlHash(false)
    , BroadcastFetch(false)
    , SkipSameDocids(false)
    , LockIndexFiles(false)
    , LockUrl2DocId(true)
    , TwoStepQuery(true)
    , SearchPath({"", "yandsearch"})
    , Owner(owner)
    , UseRTYExtensions(true)
    , ExternalSearch("")
    , CustomBaseSearch("")
    , ReArrangeOptions("")
    , EventLog("")
    , AsyncSearch(true)
    , TiersCount(1)
    , TierMinSize(10000000 * 1.5)
    , TiersOverlapPercent(0)
    , RawPassages(false)
    , FactorsInfo("")
    , KeepAllDocuments(true)
    , FiltrationModel(fmWeb)
    , FactorsModels()
    , ReAskBaseSearches(false)
    , ArchiveCacheSizeMb(0)
    , PageScanSize(10000)
    , LocalSearcherHost("127.0.0.1")
    , PrefetchSizeBytes(0)
    , GroupPruningCoefficient(1000)
    , AllowEmptyNehReplier(false)
{
    CacheOptions.UserTimeout = 0;
}

TSearcherConfig::~TSearcherConfig() = default;

void TSearcherConfig::InitCacheOptions(TCacheOptions& cacheOptions,  const TYandexConfig::Directives& directives) {
    TYandexConfig dummy;
    AssertCorrectConfig(directives.Value("LifeTime", -1) == -1, "LifeTime in Cache is incorrect, use CacheLifeTime");
    TYandexConfig::Directives::const_iterator iter = directives.find("CacheLifeTime");
    if (iter != directives.end())
        AssertCorrectConfig(TStringBuf(iter->second).find(':') == TCiString::npos, "Do not use UpdatePeriod in CachLifeTime, it will be set to zero");
    cacheOptions.Init(dummy, directives, NFs::CurrentWorkingDirectory() + GetDirectorySeparator());
    cacheOptions.UserTimeout = 0;
    cacheOptions.CacheLifeTime.UpdatePeriod = TDuration::Zero();
}

void TSearcherConfig::Init(TYandexConfig& yandexConfig, const TYandexConfig::Section* section, TConfigPreprocessor *configPreprocessor) {
    Init(section->GetDirectives(), configPreprocessor);
    TYandexConfig::TSectionsMap sectionsMap = section->GetAllChildren();
    TYandexConfig::TSectionsMap::const_iterator end = sectionsMap.end();
    TYandexConfig::TSectionsMap::const_iterator iter = sectionsMap.find("QueryLanguage");
    if (iter != end) {
        ReqAttrList.Reset(new TReqAttrList(yandexConfig, *iter->second, false));
        QueryLanguage.clear();
        for (TYandexConfig::Directives::const_iterator directivesIter = iter->second->GetDirectives().begin(); directivesIter != iter->second->GetDirectives().end(); ++directivesIter) {
            QueryLanguage += TString(directivesIter->first) + " :" + directivesIter->second + "\n";
        }
    }

    iter = sectionsMap.find("QueryCache");
    if (iter != end) {
        InitCacheOptions(CacheOptions, iter->second->GetDirectives());
    } else {
        CacheOptions.UseCache = false;
    }

    iter = sectionsMap.find("FastCache");
    if (iter != end) {
        InitCacheOptions(FastCacheOptions, iter->second->GetDirectives());
    } else {
        FastCacheOptions.UseCache = false;
    }

    iter = sectionsMap.find("TextMachine");
    if (iter != end) {
        TextMachine.DefaultTextMachine = iter->second->GetDirectives().Value("DefaultTextMachine", TString());
        TextMachine.TitleSentencesNumber = iter->second->GetDirectives().Value<ui32>("TitleSentencesNumber", 0);
        TextMachine.TitleZones = iter->second->GetDirectives().Value("TitleZones", TString());
        StringSplitter(TextMachine.TitleZones).SplitBySet(", ").SkipEmpty().Collect(&TextMachine.TitleZonesSet);
    }

    iter = sectionsMap.find("HttpOptions");
    AssertCorrectConfig(iter != end, "Searcher section must contain HttpOptions section");
    ServerOptions.Init(iter->second->GetDirectives());
}

void TSearcherConfig::AddDefaultConfigInstance(const TString& configPath, TSearcherConfig::TSearchConfigType searcherType, TConfigPreprocessor *configPreprocessor) {
    if (!!configPath) {
        AssertCorrectConfig(NFs::Exists(configPath), "Cannot find search config %s", configPath.data());
        THolder<TSearchConfig> searchConfig(new TSearchConfig);
        try {
            TAnyYandexConfig yc;
            bool parsed = searchConfig->Load(configPath, TString(), yc, configPreprocessor);
            //SearchConfig and YandexConfig errors are reported to SEARCH_ERROR
            AssertCorrectConfig(parsed, "Cannot parse %s as SearchConfig", configPath.data());
        } catch(const yexception& e) {
            Y_FAIL("SearcherConfig preprocessing error: %s", e.what());
        }

        searchConfig->SetFullDump(searchConfig->Dump());
        DefaultConfigInstances[searcherType] = searchConfig.Release();
    } else {
        DefaultConfigInstances[searcherType] = nullptr;
    }
}

void TSearcherConfig::Init(const TYandexConfig::Directives& directives, TConfigPreprocessor *configPreprocessor) {
    GET_VALUE(AutoStartServer);
    GET_VALUE(ExceptionOnSearch);
    GET_VALUE(DelegateRequestOptimization);
    GET_VALUE(Enabled);
    GET_VALUE(AccessLog);
    Owner.DaemonConfig.StartLogging<TRTYAccessLog>(AccessLog);
    GET_VALUE(EnableUrlHash);
    GET_VALUE(BroadcastFetch);
    GET_VALUE(SkipSameDocids);
    GET_VALUE_DEF(MergeMultiplier);
    GET_VALUE(LockIndexFiles);
    GET_VALUE(LockUrl2DocId);
    GET_VALUE(TwoStepQuery);
    GET_VALUE_DEF(ProtocolType);
    GET_VALUE(RetryStartMetasearch);
    GET_VALUE(LocalHostName);
    GET_VALUE(RawPassages);
    GET_VALUE(FactorsInfo);
    GET_VALUE(FactorsModels);
    GET_VALUE(KeepAllDocuments);
    GET_VALUE_DEF(CustomSearcher);
    GET_VALUE_DEF(FiltrationModel);
    GET_VALUE(SnippetsDeniedZones);
    StringSplitter(SnippetsDeniedZones).SplitBySet(", ").SkipEmpty().Collect(&SnippetsDeniedZonesVector);

    GET_VALUE(EventLog);
    GET_VALUE_DEF(AsyncSearch);
    {
        TString LoadLog = "";
        GET_VALUE(LoadLog);
        if (!!LoadLog) {
            LoadLogMeta = LoadLog;
        }
    }
    GET_VALUE_DEF(LoadLogMeta);
    GET_VALUE(LoadLogBase);
    GET_VALUE(RedirectSearchErrorLogToGlobal);
    {
        TString PassageLog = "";
        GET_VALUE(PassageLog);
        if (!!PassageLog) {
            PassageLogMeta = PassageLog;
        }
    }
    GET_VALUE_DEF(PassageLogMeta);
    GET_VALUE(PassageLogBase);

    GET_VALUE_DEF(ArchivePolicy);
    if (!!ArchivePolicy && ArchivePolicy != ARCHIVE_POLICY_DISK && ArchivePolicy != ARCHIVE_POLICY_INMEM && ArchivePolicy != ARCHIVE_POLICY_INMEMLOCK &&
            ArchivePolicy != ARCHIVE_POLICY_MAPMEM && ArchivePolicy != ARCHIVE_POLICY_MAPMEMLOCK && ArchivePolicy != ARCHIVE_POLICY_DISK_NO_CACHE)
        FAIL_LOG("Incorrect ArchivePolicy value");

    TString archiveType;
    GET_VALUE_DEF(archiveType);
    if (archiveType.empty())
        archiveType = ::ToString(AT_FLAT);

    AssertCorrectConfig(TryFromString(archiveType, ArchiveType),
                                        "%s %s", "Unknown ArchiveType",
                                        archiveType.data());
    {
        SearchPath = SplitString(directives.Value<TString>("SearchPath", JoinSeq(",", SearchPath)), ",", 0, KEEP_EMPTY_TOKENS);
        if (SearchPath.empty()) {
            SearchPath.push_back({});
        }
    }

    GET_VALUE(TiersCount);
    GET_VALUE(TierMinSize);
    GET_VALUE(TiersOverlapPercent);

    GET_VALUE(ScatterTimeout);
    GET_VALUE(Limits);
    GET_VALUE(RequestLimits);
    GET_VALUE(ReportMethod);
    GET_VALUE(ReportModule);
    GET_VALUE(WildcardSearch);
    GET_VALUE(UseRTYExtensions);
    GET_VALUE(ReArrangeOptions);
    GET_VALUE(ReAskBaseSearches);
    GET_VALUE(ArchiveCacheSizeMb);
    GET_VALUE(PageScanSize);
    AssertCorrectConfig(PageScanSize < MAX_GROUP_COUNT, "PageScanSize should be less than %d ", MAX_GROUP_COUNT);
    GET_VALUE_DEF(LocalSearcherHost);
    GET_VALUE(PrefetchSizeBytes);
    GET_VALUE(GroupPruningCoefficient);

    GET_VALUE_DEF(ExternalSearch);
    GET_VALUE_DEF(CustomBaseSearch);
    GET_VALUE_DEF(DefaultBaseSearchConfig);
    GET_VALUE_DEF(DefaultMemorySearchConfig);
    GET_VALUE_DEF(DefaultMetaSearchConfig);
    GET_VALUE(AdditionalLockedFiles);
    GET_VALUE(AdditionalPrefetchedFiles);
    GET_VALUE(WarmupQueries);
    GET_VALUE(AllowEmptyNehReplier);
    GET_VALUE(EnableCacheForDbgRlv);
    GET_VALUE(AskFactorsOnSearchStage);
    DefaultConfigInstances.resize(sctCount, TAtomicSharedPtr<TSearchConfig>(nullptr));
    AddDefaultConfigInstance(DefaultBaseSearchConfig, sctBase, configPreprocessor);
    AddDefaultConfigInstance(DefaultMemorySearchConfig, sctMemory, configPreprocessor);
    AddDefaultConfigInstance(DefaultMetaSearchConfig, sctMeta, configPreprocessor);

    try {
        const TFsPath factorsPath { FactorsInfo };
        const TString& factorsFileName = factorsPath.GetName();
        Factors = new NRTYFactors::TConfig(FactorsInfo.data(), FactorsModels, configPreprocessor, Owner.ConfigsVersions.Value(factorsFileName, -1));
        Owner.Pruning.Reset(TPruningConfig::Create(Owner.Pruning->ToString()));
    } catch(yexception&) {
        AssertCorrectConfig(false, "in factors config: %s", CurrentExceptionMessage().data());
    }
}

TString TSearcherConfig::ToString() const {
    TString s;
    TStringOutput so(s);

    so << "<Searcher>" << Endl
       << "AutoStartServer : " << AutoStartServer << Endl
       << "ExceptionOnSearch: " << ExceptionOnSearch << Endl
       << "DelegateRequestOptimization : " << DelegateRequestOptimization << Endl
       << "Enabled : " << Enabled << Endl
       << "AccessLog : " << AccessLog << Endl
       << "EnableUrlHash : " << EnableUrlHash << Endl
       << "BroadcastFetch : " << BroadcastFetch << Endl
       << "SkipSameDocids : " << SkipSameDocids << Endl
       << "MergeMultiplier : " << MergeMultiplier << Endl
       << "LockIndexFiles : " << LockIndexFiles << Endl
       << "LockUrl2DocId : " << LockUrl2DocId << Endl
       << "SnippetsDeniedZones : " << SnippetsDeniedZones << Endl
       << "EventLog : " << EventLog << Endl
       << "AsyncSearch : " << AsyncSearch << Endl
       << "LoadLogBase : " << LoadLogBase << Endl
       << "LoadLogMeta : " << LoadLogMeta << Endl
       << "RedirectSearchErrorLogToGlobal : " << RedirectSearchErrorLogToGlobal << Endl
       << "PassageLogMeta : " << PassageLogMeta << Endl
       << "PassageLogBase : " << PassageLogBase << Endl
       << "ArchivePolicy : " << ArchivePolicy << Endl
       << "ArchiveType : " << ArchiveType << Endl
       << "TiersCount : " << TiersCount << Endl
       << "TierMinSize : " << TierMinSize << Endl
       << "TiersOverlapPercent : " << TiersOverlapPercent << Endl
       << "TwoStepQuery : " << TwoStepQuery << Endl
       << "ProtocolType : " << ProtocolType << Endl
       << "RetryStartMetasearch : " << RetryStartMetasearch << Endl
       << "LocalHostName : " << LocalHostName << Endl
       << "RawPassages : " << RawPassages << Endl
       << "FactorsInfo : " << FactorsInfo << Endl
       << "FactorsModels : " << FactorsModels << Endl
       << "KeepAllDocuments : " << KeepAllDocuments << Endl
       << "FiltrationModel : " << FiltrationModel << Endl
       << "CustomSearcher : " << CustomSearcher << Endl
       << "SearchPath : " << JoinSeq(",", SearchPath) << Endl
       << "ScatterTimeout : " << ScatterTimeout << Endl
       << "Limits : " << Limits << Endl
       << "RequestLimits : " << RequestLimits << Endl
       << "ReportMethod : " << ReportMethod << Endl
       << "ReportModule : " << ReportModule << Endl
       << "WildcardSearch : " << WildcardSearch << Endl
       << "UseRTYExtensions : " << UseRTYExtensions << Endl
       << "ExternalSearch : " << ExternalSearch << Endl
       << "CustomBaseSearch : " << CustomBaseSearch << Endl
       << "ReArrangeOptions : " << ReArrangeOptions << Endl
       << "ReAskBaseSearches : " << ReAskBaseSearches << Endl
       << "ArchiveCacheSizeMb : " << ArchiveCacheSizeMb << Endl
       << "PageScanSize : " << PageScanSize << Endl
       << "LocalSearcherHost : " << LocalSearcherHost << Endl
       << "PrefetchSizeBytes : " << PrefetchSizeBytes << Endl
       << "DefaultBaseSearchConfig : " << DefaultBaseSearchConfig << Endl
       << "DefaultMemorySearchConfig : " << DefaultMemorySearchConfig << Endl
       << "DefaultMetaSearchConfig : " << DefaultMetaSearchConfig << Endl
       << "AdditionalLockedFiles : " << AdditionalLockedFiles << Endl
       << "AdditionalPrefetchedFiles : " << AdditionalPrefetchedFiles << Endl
       << "WarmupQueries : " << WarmupQueries << Endl
       << "GroupPruningCoefficient : " << GroupPruningCoefficient << Endl
       << "AllowEmptyNehReplier : " << AllowEmptyNehReplier << Endl
       << "EnableCacheForDbgRlv : " << EnableCacheForDbgRlv << Endl
       << "AskFactorsOnSearchStage : " << AskFactorsOnSearchStage << Endl
       << Endl << "<QueryLanguage>" << Endl << QueryLanguage << "</QueryLanguage>"<< Endl;

    if (CacheOptions.UseCache) {
        so << Endl << "<QueryCache>" << Endl
           << ToStringCacheOptions(CacheOptions) << Endl
           << "</QueryCache>" << Endl;
    }

    if (FastCacheOptions.UseCache) {
        so << Endl << "<FastCache>" << Endl
           << ToStringCacheOptions(FastCacheOptions) << Endl
           << "</FastCache>" << Endl;
    }

    {
        so << "<TextMachine>" << Endl;
        so << "DefaultTextMachine : " << TextMachine.DefaultTextMachine << Endl;
        so << "TitleSentencesNumber : " << TextMachine.TitleSentencesNumber << Endl;
        so << "TitleZones : " << TextMachine.TitleZones << Endl;
        so << "</TextMachine>" << Endl;
    }

    so << ServerOptions.ToString("HttpOptions") << Endl;
    so << "</Searcher>" << Endl;
    return s;
}

TString TSearcherConfig::ToStringCacheOptions(const TCacheOptions& cacheOptions) const {
    TString s;
    TStringOutput so(s);

    so << "Dir : " << cacheOptions.CacheDir << Endl
       << "Arenas : " << cacheOptions.ProtoConfig_.GetArenas() << Endl
       << "MemoryLimit : " << cacheOptions.MemoryLimit << Endl
       << "BlockSize : " << cacheOptions.BlockSize << Endl
       << "FileCacherSize : " << cacheOptions.FileCacherSize << Endl
       << "NonBlockingCacherEnabled : " << cacheOptions.ProtoConfig_.GetNonBlockingCacherEnabled() << Endl
       << "NonBlockingCacherMaxLoaderStreams : " << cacheOptions.ProtoConfig_.GetNonBlockingCacherMaxLoaderStreams() << Endl
       << "NonBlockingCacherStorerThreads : " << cacheOptions.ProtoConfig_.GetNonBlockingCacherStorerThreads() << Endl
       << "NonBlockingCacherStorerQueueSize : " << cacheOptions.ProtoConfig_.GetNonBlockingCacherStorerQueueSize() << Endl;
    if (cacheOptions.CacheLifeTime.Enabled())
        so << "CacheLifeTime : " << cacheOptions.CacheLifeTime.ValidityPeriod.Seconds() << "s" << Endl;
    so << "MinCost : " << cacheOptions.ProtoConfig_.GetMinCostToCache() << Endl
       << "ClearOnStart : " << cacheOptions.ProtoConfig_.GetClearOnStart() << Endl
       << "CompressionLevel : " << cacheOptions.ProtoConfig_.GetCompressionLevel() << Endl
       << "FileCacherSize : " << cacheOptions.FileCacherSize;
    return s;
}

bool TSearcherConfig::DelegationIsActive() const {
    return (Owner.Shards->Number == 1) &&
        (Owner.SearchersCountLimit == 1) && !Owner.RealtimeEnabled() &&
        DelegateRequestOptimization;
}

const TString& TSearcherConfig::GetPersistentSearchDir() const {
    return Owner.RealmListConfig->GetMainRealmConfig().RealmDirectory;
}

}
