#include "config.h"

#include <search/config/virthost.h>
#include <saas/util/network/address.h>

#include <library/cpp/dns/cache.h>

#include <util/string/cast.h>

namespace {
    class TConsistentClientOptionsParser
        : public TKeyValueParser<TConsistentClientOptionsParser>
        , public NFusion::TConsistentClientOptions
    {
    public:
        TConsistentClientOptionsParser(const TString& options) {
            DoParse(options);
        }
        bool TreatKeyValue(const TStringBuf& key, const TStringBuf& value) {
            if (key == "Capacity") {
                Capacity = FromString<ui32>(value);
                return true;
            }
            if (key == "EnableDynamicSwitch") {
                DynamicSwitch = IsTrue(value);
                return true;
            }
            if (key == "MaxPrimaryReplicas") {
                MaxPrimaryReplicas = FromString<ui32>(value);
                return true;
            }
            if (key == "MaxResponseChunks") {
                MaxResponseChunks = FromString<ui32>(value);
                return true;
            }
            if (key == "RestrictEmptyCacheSwitch") {
                RestrictEmptyCacheSwitch = IsTrue(value);
                return true;
            }

            return false;
        }
    };

    TString PrintConsistentClientOptions(const NFusion::TConsistentClientOptions& options) {
        TStringStream s;
        s << "Capacity=" << options.Capacity << ",";
        s << "EnableDynamicSwitch=" << options.DynamicSwitch << ",";
        s << "MaxPrimaryReplicas=" << options.MaxPrimaryReplicas << ",";
        if (options.MaxResponseChunks)  // an experimental options, so it is hidden if not set
            s << "MaxResponseChunks=" << options.MaxPrimaryReplicas << ",";
        s << "RestrictEmptyCacheSwitch=" << options.RestrictEmptyCacheSwitch;
        return s.Str();
    }

    bool Resolvable(const NRealTime::TDistributor& distributor, NBus::EIpVersion required, NBus::EIpVersion preferred) {
        auto na = NUtil::Resolve(distributor.Host, distributor.Port);
        if (!na.Defined()) {
            ERROR_LOG << "Cannot resolve " << distributor << Endl;
            return false;
        }

        try {
            NBus::TNetAddr specified(na.GetRef()->Addr, required, preferred);
        } catch (const yexception& e) {
            ERROR_LOG << distributor << " " << e.what() << Endl;
            return false;
        }
        return true;
    }

    bool Resolvable(const NFusion::TConsistentClientReplica& replica, NBus::EIpVersion required, NBus::EIpVersion preferred) {
        for (auto&& distributor: replica) {
            if (!Resolvable(distributor, required, preferred)) {
                return false;
            }
        }

        return true;
    }
}

namespace {
    void ParseDuration(const TYandexConfig::Directives& dirs, const TString& lable, TDuration& result) {
        if (dirs.contains(lable)) {
            TString value;
            dirs.GetValue(lable, value);
            VERIFY_WITH_LOG(TDuration::TryParse(value, result), "Failed to parse %s: %s", lable.data(), value.data());
        }
    }
}


namespace NFusion {
void TDocFetcherConfig::DoInit(const TYandexConfig::Section& moduleSection) {
    const TYandexConfig::Directives& directives = moduleSection.GetDirectives();

    directives.GetValue("Enabled", Enabled);
    directives.GetValue("LogFile", LogFile);
    directives.GetValue("SysLogFile", SysLogFile);
    directives.GetValue("WatchdogOptionsFile", WatchdogOptionsFile);
    directives.GetValue("SearchOpenDocAgeSec", SearchOpenDocAgeSec);
    directives.GetValue("EnableSearchOnStart", EnableSearchOnStart);

    for (const auto& sec: moduleSection.GetAllChildren()) {
        if (sec.first == "DatacenterChecker") {
            auto type = GetStreamType(sec.second->GetDirectives());
            switch (type) {
            case NFusion::EStreamType::PersQueue: {
                VERIFY_WITH_LOG(!PQDatacenterChecker.Defined(), "Only one DatacenterChecker per stream type is allowed");
                ParseDatacenterCheckerConfig(*sec.second, PQDatacenterChecker.ConstructInPlace());
                break;
            }
            default:
                VERIFY_WITH_LOG(false, "Unsupported stream type for DatacenterChecker");
            }
            continue;
        }

        if (sec.first != "Stream")
            continue;

        auto type = GetStreamType(sec.second->GetDirectives());
        switch (type) {
        case NFusion::EStreamType::OldDistributor: {
            TStreamConfig stream;
            ParseStreamConfig(*sec.second, stream);
            Streams.push_back(stream);
            break;
        }
        case NFusion::EStreamType::Distributor: {
            TStreamConfig stream;
            ParseStreamConfig(*sec.second, stream);
            DistributorStreams.push_back(stream);
            break;
        }
        case NFusion::EStreamType::MapReduce: {
            TMapReduceStreamConfig stream;
            ParseStreamConfig(*sec.second, stream);
            MapReduceStreams.push_back(stream);
            break;
        }
        case NFusion::EStreamType::PersQueue: {
            TPersQueueStreamConfig stream;
            ParseStreamConfig(*sec.second, stream);
            PersQueueStreams.push_back(stream);
            break;
        }
        case NFusion::EStreamType::Snapshot: {
            TSnapshotStreamConfig stream;
            ParseStreamConfig(*sec.second, stream);
            if (stream.Enabled) {
                VERIFY_WITH_LOG(!SnapshotStream.Defined(), "Only one Snapshot stream is allowed");
                SnapshotStream.ConstructInPlace(stream);
            }
            break;
        }
        default:
            FAIL_LOG("Unexpected behaviour");
        }
    }
}

bool TDocFetcherConfig::DoCheck() const {
    TSet<TString> names;
    TSet<ui32> ids;
    for (auto&& stream : PersQueueStreams) {
        if (stream.Enabled) {
            VERIFY_WITH_LOG(names.insert(stream.Name).second, "DocFetcher: Stream names should be unique");
            VERIFY_WITH_LOG(ids.insert(stream.StreamId).second, "DocFetcher: Stream IDs should be unique");
        }
    }
    for (auto&& stream : MapReduceStreams) {
        if (stream.Enabled) {
            VERIFY_WITH_LOG(names.insert(stream.Name).second, "DocFetcher: Stream names should be unique");
            VERIFY_WITH_LOG(ids.insert(stream.StreamId).second, "DocFetcher: Stream IDs should be unique");
        }
    }
    for (auto&& stream : Streams) {
        if (stream.Enabled) {
            VERIFY_WITH_LOG(names.insert(stream.Name).second, "DocFetcher: Stream names should be unique");
            VERIFY_WITH_LOG(ids.insert(stream.StreamId).second, "DocFetcher: Stream IDs should be unique");
        }
    }
    if (SnapshotStream && SnapshotStream->Enabled) {
        VERIFY_WITH_LOG(names.insert(SnapshotStream->Name).second, "DocFetcher: Stream names should be unique");
        VERIFY_WITH_LOG(ids.insert(SnapshotStream->StreamId).second, "DocFetcher: Stream IDs should be unique");
    }
    for (auto&& name : names) {
        VERIFY_WITH_LOG(name, "DocFetcher: Stream name should be non-empty");
    }
    return true;
}

void TDocFetcherConfig::DoToString(IOutputStream& so) const {
    so << "Enabled : " << Enabled << Endl;
    so << "LogFile : " << LogFile << Endl;
    so << "SysLogFile : " << SysLogFile << Endl;
    so << "WatchdogOptionsFile : " << WatchdogOptionsFile << Endl;
    so << "SearchOpenDocAgeSec : " << SearchOpenDocAgeSec << Endl;
    so << "EnableSearchOnStart : " << EnableSearchOnStart << Endl;

    if (PQDatacenterChecker) {
        so << "<DatacenterChecker>" << Endl;
        PrintDatacenterCheckerConfig(*PQDatacenterChecker, so);
        so << "</DatacenterChecker>" << Endl;
    }

    for (const auto& stream: Streams) {
        so << "<Stream>" << Endl;
        PrintStreamConfig(stream, so);
        so << "</Stream>" << Endl;
    }
    for (const auto& stream : DistributorStreams) {
        so << "<Stream>" << Endl;
        PrintStreamConfig(stream, so);
        so << "</Stream>" << Endl;
    }
    for (const auto& stream : MapReduceStreams) {
        so << "<Stream>" << Endl;
        PrintStreamConfig(stream, so);
        so << "</Stream>" << Endl;
    }
    for (const auto& stream : PersQueueStreams) {
        so << "<Stream>" << Endl;
        PrintStreamConfig(stream, so);
        so << "</Stream>" << Endl;
    }
    if (SnapshotStream) {
        so << "<Stream>" << Endl;
        PrintStreamConfig(*SnapshotStream, so);
        so << "</Stream>" << Endl;
    }
}

NFusion::EStreamType TDocFetcherConfig::GetStreamType(const TYandexConfig::Directives& directives) const {
    NFusion::EStreamType result = NFusion::EStreamType::Distributor;
    directives.GetValue("StreamType", result);
    return result;
}

void TDocFetcherConfig::ParseCommonStreamConfig(const TYandexConfig::Directives& dirs, TBaseStreamConfig& stream) const {
    dirs.GetValue("Name", stream.Name);
    dirs.GetValue("StreamId", stream.StreamId);
    dirs.GetValue("StreamType", stream.StreamType);
    dirs.GetValue("ShardMin", stream.ShardMin);
    dirs.GetValue("ShardMax", stream.ShardMax);
    CHECK_WITH_LOG(stream.ShardMin <= stream.ShardMax);
    dirs.GetValue("FilterShardMin", stream.ShardFilter.ShardMin);
    dirs.GetValue("FilterShardMax", stream.ShardFilter.ShardMax);
    dirs.GetValue("FilterShardType", stream.ShardFilter.ShardType);
    dirs.GetValue("FilterShardKpsShift", stream.ShardFilter.KpsShift);
    if (stream.ShardFilter.IsEnabled()) {
        CHECK_WITH_LOG(stream.ShardFilter.ShardMin <= stream.ShardFilter.ShardMax);
        CHECK_WITH_LOG(stream.ShardMin <= stream.ShardFilter.ShardMin);
        CHECK_WITH_LOG(stream.ShardFilter.ShardMax <= stream.ShardMax);
    }
    ParseDuration(dirs, "PauseOnMergeDuration", stream.PauseOnMergeDuration);
    dirs.GetValue("Shard", stream.Shard);
    dirs.GetValue("NumShards", stream.NumShards);
    dirs.GetValue("UseIndexTimestamp", stream.UseIndexTimestamp);
    dirs.GetValue("AsyncStart", stream.AsyncStart);
    dirs.GetValue("Enabled", stream.Enabled);
    dirs.GetValue("PauseOnMerge", stream.PauseOnMerge);
    dirs.GetValue("YTHosts", stream.YTHosts);
}

void TDocFetcherConfig::ParseDocStreamConfig(const TYandexConfig::Directives& dirs, TDocStreamConfig& stream) const {
    ParseCommonStreamConfig(dirs, stream);

    dirs.GetValue("MaxAgeToGetSec", stream.MaxAgeToGetSec);
    dirs.GetValue("MaxDocAgeToKeepSec", stream.MaxDocAgeToKeepSec);
    dirs.GetValue("MemIndexAgeSec", stream.MemIndexAgeSec);
    dirs.GetValue("MemoryIndexDistAgeThreshold", stream.MemoryIndexDistAgeThreshold);

    dirs.GetValue("DistAgeAsDocAge", stream.ReceiveDurationAsDocAge);
    dirs.GetValue("ReceiveDurationAsDocAge", stream.ReceiveDurationAsDocAge);

    if (dirs.contains("SearchOpenThreshold")) {
        dirs.GetValue("SearchOpenThreshold", stream.SearchOpenThreshold);
    }
    else if (SearchOpenDocAgeSec != -1) {
        stream.SearchOpenThreshold = SearchOpenDocAgeSec;
    }

    ParseDuration(dirs, "TimeToSleep", stream.TimeToSleep);
    ParseDuration(dirs, "MaxIdleTime", stream.MaxIdleTime);

    dirs.GetValue("SearchOpenThresholdPessimistic", stream.SearchOpenThresholdPessimistic);
    dirs.GetValue("SearchOpenPessimizationDelaySeconds", stream.SearchOpenPessimizationDelaySeconds);
    CHECK_WITH_LOG(stream.SearchOpenThreshold >= stream.SearchOpenThresholdPessimistic) << stream.SearchOpenThreshold << " >= " << stream.SearchOpenThresholdPessimistic;

    TString distributionMetrics;
    dirs.GetValue("DistributionMetrics", distributionMetrics);
    SplitString(distributionMetrics.data(), distributionMetrics.data() + distributionMetrics.size(), TCharDelimiter<const char>(','),
                TContainerConsumer<TSet<TString>>(&stream.DistributionMetrics));

    dirs.GetValue("Timeout", stream.ClientOptions.Timeout);
    dirs.GetValue("InterAttemptTimeout", stream.ClientOptions.InterAttemptTimeout);
    dirs.GetValue("Threads", stream.ClientOptions.Threads);
    dirs.GetValue("Attempts", stream.ClientOptions.Attempts);
    dirs.GetValue("Rate", stream.ClientOptions.AverageRate);
    dirs.GetValue("BurstRate", stream.ClientOptions.BurstRate);
    dirs.GetValue("IndexingHost", stream.IndexingHost);
    dirs.GetValue("IndexingPort", stream.IndexingPort);

    dirs.GetValue("RealtimeOverride", stream.RealtimeOverride);
    dirs.GetValue("FastIndexWhenNoSearch", stream.FastIndexWhenNoSearch);
}

void TDocFetcherConfig::ParseStreamConfig(const TYandexConfig::Section& section, TStreamConfig& stream) const {
    const auto& dirs = section.GetDirectives();
    const auto& sections = section.GetAllChildren();

    ParseDocStreamConfig(dirs, stream);
    ParseSyncConfig(section, stream);
    dirs.GetValue("UseCompression", stream.UseCompression);
    dirs.GetValue("ConsistentClient", stream.ConsistentClient);
    dirs.GetValue("DistributorStream", stream.DistributorStream);
    dirs.GetValue("DistributorAttributes", stream.DistributorAttributes);
    dirs.GetValue("OverlapAge", stream.OverlapAge);
    dirs.GetValue("MaxUnresolvedReplicas", stream.MaxUnresolvedReplicas);
    dirs.GetValue("WarnUnresolvedReplicas", stream.WarnUnresolvedReplicas);
    dirs.GetValue("ClientId", stream.ClientId);

    VERIFY_WITH_LOG(stream.Shard < stream.NumShards, "DocFetcher: Shard number %d is not in the expected range of [0..%d)", stream.Shard, stream.NumShards);

    TString distributorServers;
    TString preferredDistributor;
    dirs.GetValue("DistributorServers", distributorServers);
    dirs.GetValue("PreferredDistributor", preferredDistributor);
    stream.DistributorServers = NRealTime::ParseDistributorString(distributorServers);
    stream.PreferredDistributor = NRealTime::ParseDistributorString(preferredDistributor);
    stream.DistributorReplicas = ParseClientReplicas(sections, stream);

    TString options;
    dirs.GetValue("ConsistentClientOptions", options);
    stream.ConsistentClientOptions = ParseClientOptions(options);

    stream.DistributorServers = PreresolveAndFilter(stream.DistributorServers);
    stream.PreferredDistributor = PreresolveAndFilter(stream.PreferredDistributor);
}

void TDocFetcherConfig::ParseStreamConfig(const TYandexConfig::Section& section, TMapReduceStreamConfig& stream) const {
    const auto& dirs = section.GetDirectives();

    ParseDocStreamConfig(dirs, stream);
    ParseSyncConfig(section, stream);

    dirs.GetValue("Server", stream.Server);
    dirs.GetValue("Token", stream.Token);
    dirs.GetValue("Table", stream.Table);
    dirs.GetValue("BackupTable", stream.BackupTable);
    dirs.GetValue("RowProcessor", stream.RowProcessor);
    dirs.GetValue("MasterTableCheckInterval", stream.MasterTableCheckInterval);
    dirs.GetValue("BackupSizeThresholdPercent", stream.BackupSizeThresholdPercent);
    if (!stream.SyncPath.empty()) {
        VERIFY_WITH_LOG(stream.BackupTable.empty(), "BackupTable is not compatible with TSyncStream");
    }
}

void TDocFetcherConfig::ParseStreamConfig(const TYandexConfig::Section& section, TPersQueueStreamConfig& stream) const {
    const auto& dirs = section.GetDirectives();
    const auto& sections = section.GetAllChildren();

    ParseDocStreamConfig(dirs, stream);
    ParseSyncConfig(section, stream);

    dirs.GetValue("Server", stream.Server);
    dirs.GetValue("Ident", stream.Ident);
    dirs.GetValue("TopicName", stream.TopicName);

    if (dirs.contains("NumTopics")) {
        dirs.GetValue("NumTopics", stream.NumTopics);
    } else {
        stream.NumTopics = stream.NumShards;
    }

    dirs.GetValue("UseShardedLogtype", stream.UseShardedLogtype);
    dirs.GetValue("UserPrefix", stream.UserPrefix);
    dirs.GetValue("Consumer", stream.Consumer);
    dirs.GetValue("BatchSize", stream.BatchSize);
    dirs.GetValue("BatchInBytes", stream.BatchInBytes);
    dirs.GetValue("Timeout", stream.Timeout);
    dirs.GetValue("ReadTimeout", stream.ReadTimeout);
    dirs.GetValue("LagTimeout", stream.LagTimeout);
    dirs.GetValue("SleepTimeout", stream.SleepTimeout);
    ParseDuration(dirs, "StartBlockTime", stream.StartBlockTime);
    ParseDuration(dirs, "ReceiveDelay", stream.ReceiveDelay);
    dirs.GetValue("QueueSize", stream.QueueSize);
    dirs.GetValue("UseIpv6", stream.UseIpv6);
    dirs.GetValue("UseNewProtocol", stream.UseNewProtocol);
    dirs.GetValue("UseNewPQLib", stream.UseNewPQLib);
    dirs.GetValue("UseDirectoryTopicFormat", stream.UseDirectoryTopicFormat);
    dirs.GetValue("PQLibThreads", stream.PQLibThreads);
    dirs.GetValue("Replica", stream.Replica);
    dirs.GetValue("StreamRead", stream.StreamRead);
    dirs.GetValue("UseMirroredPartitions", stream.UseMirroredPartitions);
    dirs.GetValue("Datacenters", stream.Datacenters);
    dirs.GetValue("OverlapAge", stream.OverlapAge);
    dirs.GetValue("LockServers", stream.LockServers);
    dirs.GetValue("LockPath", stream.LockPath);
    stream.ReplicaIdsList = ParseReplicaIds(sections);
    TYandexConfig::TSectionsMap::const_iterator tvmSection = sections.find("TVM");
    if (tvmSection != sections.end()) {
        stream.TvmConfig.ConstructInPlace().Init(*tvmSection->second);
    }
}

void TDocFetcherConfig::ParseStreamConfig(const TYandexConfig::Section& section, TSnapshotStreamConfig& stream) const {
    const auto& dirs = section.GetDirectives();
    const auto& sections = section.GetAllChildren();

    ParseCommonStreamConfig(dirs, stream);
    ParseSyncConfig(section, stream);

    dirs.GetValue("Mode", stream.Mode);

    stream.IterationsPause = TDuration::Seconds(dirs.Value<ui64>("IterationsPause", stream.IterationsPause.Seconds()));
    stream.FetchRetryPause = TDuration::Seconds(dirs.Value<ui64>("FetchRetryPause", stream.FetchRetryPause.Seconds()));
    stream.PauseAfterPartialConsume = TDuration::Seconds(dirs.Value<ui64>("PauseAfterPartialConsume", stream.PauseAfterPartialConsume.Seconds()));
    dirs.GetValue("FetchMaxAttempts", stream.FetchMaxAttempts);
    dirs.GetValue("MaxResourcesPerIteration", stream.MaxResourcesPerIteration);

    dirs.GetValue("SnapshotManager", stream.SnapshotManager);
    dirs.GetValue("SnapshotServer", stream.SnapshotServer);
    dirs.GetValue("SnapshotPath", stream.SnapshotPath);
    dirs.GetValue("SnapshotToken", stream.SnapshotToken);
    dirs.GetValue("UseGroups", stream.UseGroups);

    stream.FixedTimestamp = TInstant::Seconds(dirs.Value<ui64>("FixedTimestampUnix", stream.FixedTimestamp.Seconds()));
    dirs.GetValue("ConsumeMode", stream.ConsumeMode);

    const auto iter = sections.find(NRTYServer::TResourceFetchConfig::SECTION_NAME);
    if (iter != sections.end() && iter->second) {
        stream.ResourceFetchConfig.ConstructInPlace();
        stream.ResourceFetchConfig->Init(*iter->second);
    }

    TString allowEmptyIndexUntilStr;
    dirs.GetValue("AllowEmptyIndexUntil", allowEmptyIndexUntilStr);
    if (!allowEmptyIndexUntilStr.empty()) {
        try {
            stream.AllowEmptyIndexUntil = TInstant::ParseIso8601(allowEmptyIndexUntilStr);
        } catch (...) {
            ERROR_LOG << "Cannot parse AllowEmptyIndexUntil = '" << allowEmptyIndexUntilStr << "', error = " << CurrentExceptionMessage() << Endl;
        }
    }
}

void TDocFetcherConfig::ParseSyncConfig(const TYandexConfig::Section& section, TSyncOptions& stream) const {
    const auto& dirs = section.GetDirectives();
    const auto& sections = section.GetAllChildren();
    const auto iter = sections.find(NRTYServer::TResourceFetchConfig::SECTION_NAME);
    if (iter != sections.end() && iter->second) {
        stream.ResourceFetchConfig.ConstructInPlace();
        stream.ResourceFetchConfig->Init(*iter->second);
    }

    dirs.GetValue("SyncServer", stream.SyncServer);
    dirs.GetValue("SyncPath", stream.SyncPath);
    dirs.GetValue("SyncMaxAttempts", stream.SyncMaxAttempts);
    dirs.GetValue("SnapshotManager", stream.SnapshotManager);
    dirs.GetValue("EnableSearchWhileFetching", stream.EnableSearchWhileFetching);
    stream.SyncThreshold = TDuration::Seconds(dirs.Value<ui64>("SyncThreshold", stream.SyncThreshold.Seconds()));
    stream.SyncAttemptPause = TDuration::Seconds(dirs.Value<ui64>("SyncAttemptPause", stream.SyncAttemptPause.Seconds()));
}

void TDocFetcherConfig::PrintCommonStreamConfig(const TBaseStreamConfig& stream, IOutputStream& so) const {
    so << "Name : " << stream.Name << Endl;
    so << "StreamId : " << stream.StreamId << Endl;
    so << "StreamType : " << stream.StreamType << Endl;
    so << "ShardMin : " << stream.ShardMin << Endl;
    so << "ShardMax : " << stream.ShardMax << Endl;
    so << "FilterShardMin : " << stream.ShardFilter.ShardMin << Endl;
    so << "FilterShardMax : " << stream.ShardFilter.ShardMax << Endl;
    so << "FilterShardType : " << stream.ShardFilter.ShardType << Endl;
    so << "FilterShardKpsShift : " << stream.ShardFilter.KpsShift << Endl;
    so << "Shard : " << stream.Shard << Endl;
    so << "NumShards : " << stream.NumShards << Endl;
    so << "PauseOnMergeDuration : " << stream.PauseOnMergeDuration.Seconds() << Endl;
    so << "UseIndexTimestamp : " << stream.UseIndexTimestamp << Endl;
    so << "AsyncStart: " << stream.AsyncStart << Endl;
    so << "Enabled : " << stream.Enabled << Endl;
    so << "PauseOnMerge : " << stream.PauseOnMerge << Endl;
}

void TDocFetcherConfig::PrintDocStreamConfig(const TDocStreamConfig& stream, IOutputStream& so) const {
    PrintCommonStreamConfig(stream, so);

    so << "MaxAgeToGetSec : " << stream.MaxAgeToGetSec << Endl;
    so << "MaxDocAgeToKeepSec : " << stream.MaxDocAgeToKeepSec << Endl;
    so << "ReceiveDurationAsDocAge : " << stream.ReceiveDurationAsDocAge << Endl;
    so << "MemIndexAgeSec : " << stream.MemIndexAgeSec << Endl;
    so << "MemoryIndexDistAgeThreshold : " << stream.MemoryIndexDistAgeThreshold << Endl;
    so << "SearchOpenThreshold : " << stream.SearchOpenThreshold << Endl;
    so << "TimeToSleep : " << stream.TimeToSleep << Endl;
    so << "MaxIdleTime : " << stream.MaxIdleTime << Endl;

    so << "SearchOpenThresholdPessimistic : " << stream.SearchOpenThresholdPessimistic << Endl;
    so << "SearchOpenPessimizationDelaySeconds : " << stream.SearchOpenPessimizationDelaySeconds << Endl;
    so << "RealtimeOverride : " << stream.RealtimeOverride << Endl;

    TString distributionMetrics = JoinStrings(stream.DistributionMetrics.begin(), stream.DistributionMetrics.end(), ",");
    so << "DistributionMetrics : " << distributionMetrics << Endl;

    so << "Timeout : " << stream.ClientOptions.Timeout << Endl;
    so << "InterAttemptTimeout : " << stream.ClientOptions.InterAttemptTimeout << Endl;
    so << "Threads : " << stream.ClientOptions.Threads << Endl;
    so << "Attempts : " << stream.ClientOptions.Attempts << Endl;
    so << "Rate : " << stream.ClientOptions.AverageRate << Endl;
    so << "BurstRate : " << stream.ClientOptions.BurstRate << Endl;
    so << "IndexingHost : " << stream.IndexingHost << Endl;
    so << "IndexingPort : " << stream.IndexingPort << Endl;
    so << "FastIndexWhenNoSearch : " << stream.FastIndexWhenNoSearch << Endl;
}

void TDocFetcherConfig::PrintStreamConfig(const TStreamConfig& stream, IOutputStream& so) const {
    PrintDocStreamConfig(stream, so);
    PrintSyncConfig(stream, so);
    PrintClientReplicas(stream.DistributorReplicas, so);
    so << "DistributorServers : " << NRealTime::SerializeDistributorString(stream.DistributorServers) << Endl;
    so << "PreferredDistributor : " << NRealTime::SerializeDistributorString(stream.PreferredDistributor) << Endl;
    so << "UseCompression : " << stream.UseCompression << Endl;
    so << "ConsistentClient : " << stream.ConsistentClient << Endl;
    so << "DistributorStream : " << stream.DistributorStream << Endl;
    so << "DistributorAttributes : " << stream.DistributorAttributes << Endl;
    so << "OverlapAge : " << stream.OverlapAge << Endl;
    so << "MaxUnresolvedReplicas : " << stream.MaxUnresolvedReplicas << Endl;
    so << "WarnUnresolvedReplicas : " << stream.WarnUnresolvedReplicas << Endl;
    so << "ClientId : " << stream.ClientId << Endl;
    so << "ConsistentClientOptions : " << PrintClientOptions(stream.ConsistentClientOptions) << Endl;
}

void TDocFetcherConfig::PrintStreamConfig(const TMapReduceStreamConfig& stream, IOutputStream& so) const {
    PrintDocStreamConfig(stream, so);
    PrintSyncConfig(stream, so);
    so << "Server : " << stream.Server << Endl;
    so << "Token : " << stream.Token << Endl;
    so << "YTHosts : " << stream.YTHosts << Endl;
    so << "Table : " << stream.Table << Endl;
    so << "BackupTable : " << stream.BackupTable << Endl;
    so << "RowProcessor: " << stream.RowProcessor << Endl;
    so << "MasterTableCheckInterval : " << stream.MasterTableCheckInterval << Endl;
    so << "BackupSizeThresholdPercent : " << stream.BackupSizeThresholdPercent << Endl;
}

void TDocFetcherConfig::PrintStreamConfig(const TPersQueueStreamConfig& stream, IOutputStream& so) const {
    PrintDocStreamConfig(stream, so);
    PrintSyncConfig(stream, so);
    so << "UseShardedLogtype : " << stream.UseShardedLogtype << Endl;
    so << "Server : " << stream.Server << Endl;
    so << "Ident : " << stream.Ident << Endl;
    so << "TopicName : " << stream.TopicName << Endl;
    so << "NumTopics : " << stream.NumTopics << Endl;
    so << "UserPrefix : " << stream.UserPrefix << Endl;
    so << "Consumer : " << stream.Consumer << Endl;
    so << "BatchSize : " << stream.BatchSize << Endl;
    so << "BatchInBytes : " << stream.BatchInBytes << Endl;
    so << "Timeout : " << stream.Timeout << Endl;
    so << "ReadTimeout : " << stream.ReadTimeout << Endl;
    so << "LagTimeout : " << stream.LagTimeout << Endl;
    so << "SleepTimeout : " << stream.SleepTimeout << Endl;
    so << "StartBlockTime : " << stream.StartBlockTime << Endl;
    so << "ReceiveDelay : " << stream.ReceiveDelay << Endl;
    so << "QueueSize : " << stream.QueueSize << Endl;
    so << "UseIpv6 : " << stream.UseIpv6 << Endl;
    so << "UseNewProtocol : " << stream.UseNewProtocol << Endl;
    so << "UseNewPQLib : " << stream.UseNewPQLib << Endl;
    so << "UseDirectoryTopicFormat : " << stream.UseDirectoryTopicFormat << Endl;
    so << "PQLibThreads : " << stream.PQLibThreads << Endl;
    so << "Replica : " << stream.Replica << Endl;
    so << "StreamRead : " << stream.StreamRead << Endl;
    so << "UseMirroredPartitions : " << stream.UseMirroredPartitions << Endl;
    so << "Datacenters : " << stream.Datacenters << Endl;
    so << "OverlapAge : " << stream.OverlapAge << Endl;
    so << "LockServers : " << stream.LockServers << Endl;
    so << "LockPath : " << stream.LockPath << Endl;
    PrintReplicaIds(stream.ReplicaIdsList, so);
    if (stream.TvmConfig) {
        so << stream.TvmConfig->ToString("TVM") << Endl;
    }
}

void TDocFetcherConfig::PrintStreamConfig(const TSnapshotStreamConfig& stream, IOutputStream& so) const {
    PrintCommonStreamConfig(stream, so);
    PrintSyncConfig(stream, so);
    so << "Mode : " << stream.Mode << Endl;
    so << "IterationsPause : " << stream.IterationsPause.Seconds() << Endl;
    so << "FetchRetryPause : " << stream.FetchRetryPause.Seconds() << Endl;
    so << "PauseAfterPartialConsume : " << stream.PauseAfterPartialConsume.Seconds() << Endl;
    so << "FetchMaxAttempts : " << stream.FetchMaxAttempts << Endl;
    so << "MaxResourcesPerIteration : " << stream.MaxResourcesPerIteration << Endl;
    so << "SnapshotManager : " << stream.SnapshotManager << Endl;
    so << "SnapshotServer : " << stream.SnapshotServer << Endl;
    so << "SnapshotPath : " << stream.SnapshotPath << Endl;
    so << "SnapshotToken : " << stream.SnapshotToken << Endl;
    so << "UseGroups : " << stream.UseGroups << Endl;
    so << "YTHosts : " << stream.YTHosts << Endl;
    so << "FixedTimestampUnix : " << stream.FixedTimestamp.Seconds() << Endl;
    so << "ConsumeMode : " << stream.ConsumeMode << Endl;
    if (stream.AllowEmptyIndexUntil == TInstant::Zero()) {
        so << "AllowEmptyIndexUntil : " << Endl;
    } else {
        so << "AllowEmptyIndexUntil : " << stream.AllowEmptyIndexUntil.ToString() << Endl;
    }

    if (stream.ResourceFetchConfig) {
        stream.ResourceFetchConfig->ToString(so);
    }
}


void TDocFetcherConfig::PrintSyncConfig(const TSyncOptions& stream, IOutputStream& so) const {
    if (stream.ResourceFetchConfig) {
        stream.ResourceFetchConfig->ToString(so);
    }
    so << "SyncServer : " << stream.SyncServer << Endl;
    so << "SyncPath : " << stream.SyncPath << Endl;
    so << "SyncMaxAttempts : " << stream.SyncMaxAttempts << Endl;
    so << "SyncThreshold : " << stream.SyncThreshold.Seconds() << Endl;
    so << "SyncAttemptPause : " << stream.SyncAttemptPause.Seconds() << Endl;
    so << "SnapshotManager : " << stream.SnapshotManager << Endl;
    so << "EnableSearchWhileFetching : " << stream.EnableSearchWhileFetching << Endl;
}


void TDocFetcherConfig::ParseDatacenterCheckerConfig(
    const TYandexConfig::Section& section,
    TDatacenterCheckerConfig& checker
) const {
    const auto& dirs = section.GetDirectives();
    dirs.GetValue("Enabled", checker.Enabled);

    dirs.GetValue("StreamType", checker.StreamType);
    dirs.GetValue("DataSourceType", checker.DataSourceType);

    if (checker.DataSourceType == EDataSourceType::ExternalAPI) {
        TExternalAPIDataSourceConfig dataSource;

        dirs.GetValue("Host", dataSource.Host);
        dirs.GetValue("Port", dataSource.Port);
        dirs.GetValue("Query", dataSource.Query);
        ParseDuration(dirs, "Timeout", dataSource.Timeout);

        checker.DataSource = dataSource;
    } else if (checker.DataSourceType == EDataSourceType::Zookeeper) {
        TZookeeperDataSourceConfig dataSource;

        dirs.GetValue("ZKPath", dataSource.ZKPath);
        dirs.GetValue("ZKServers", dataSource.ZKServers);
        dirs.GetValue("ZKRequestsMaxAttempts", dataSource.ZKRequestsMaxAttempts);
        ParseDuration(dirs, "ZKLostConnectionWaitTime", dataSource.ZKLostConnectionWaitTime);
        ParseDuration(dirs, "ZKMaxValidDataDelay", dataSource.ZKMaxValidDataDelay);

        checker.DataSource = dataSource;
    } else {
        FAIL_LOG("Unexpected data source type");
    }

    ParseDuration(dirs, "CheckInterval", checker.CheckInterval);
    ParseDuration(dirs, "BlockDuration", checker.BlockDuration);

    dirs.GetValue("MaxConsecutiveErrors", checker.MaxConsecutiveErrors);
}

void TDocFetcherConfig::PrintDatacenterCheckerConfig(const TDatacenterCheckerConfig& checker, IOutputStream& so) const {
    so << "Enabled : " << checker.Enabled << Endl;

    so << "StreamType : " << checker.StreamType << Endl;
    so << "DataSourceType : " << checker.DataSourceType << Endl;

    if (checker.DataSourceType == EDataSourceType::ExternalAPI) {
        const TExternalAPIDataSourceConfig& dataSource = std::get<TExternalAPIDataSourceConfig>(checker.DataSource);
        so << "Host : " << dataSource.Host << Endl;
        so << "Port : " << dataSource.Port << Endl;
        so << "Query : " << dataSource.Query << Endl;
        so << "Timeout : " << dataSource.Timeout << Endl;
    } else if (checker.DataSourceType == EDataSourceType::Zookeeper) {
        const TZookeeperDataSourceConfig& dataSource = std::get<TZookeeperDataSourceConfig>(checker.DataSource);
        so << "ZKPath : " << dataSource.ZKPath << Endl;
        so << "ZKServers : " << dataSource.ZKServers << Endl;
        so << "ZKRequestsMaxAttempts : " << dataSource.ZKRequestsMaxAttempts << Endl;
        so << "ZKLostConnectionWaitTime : " << dataSource.ZKLostConnectionWaitTime << Endl;
        so << "ZKMaxValidDataDelay : " << dataSource.ZKMaxValidDataDelay << Endl;
    }

    so << "CheckInterval : " << checker.CheckInterval << Endl;
    so << "BlockDuration : " << checker.BlockDuration << Endl;

    so << "MaxConsecutiveErrors : " << checker.MaxConsecutiveErrors << Endl;
}

TConsistentClientReplicas TDocFetcherConfig::ParseClientReplicas(const TYandexConfig::TSectionsMap& sections, const TStreamConfig& stream) const {
    TConsistentClientReplicas result;

    auto required  = NBus::EIP_VERSION_ANY;
    auto preferred = NBus::EIP_VERSION_6;

    ui32 resolved = 0;
    ui32 unresolved = 0;
    for (auto&& section : sections) {
        if (section.first == "Replica") {
            const auto& directives = section.second->GetDirectives();
            TConsistentClientReplica replica;

            TString servers;
            VERIFY_WITH_LOG(directives.GetValue("Servers", servers), "Missing Replica.Servers directive");
            VERIFY_WITH_LOG(directives.GetValue("Id", replica.Id), "Missing Replica.Id directive");
            VERIFY_WITH_LOG(directives.GetValue("Priority", replica.Priority), "Missing Replica.Id directive");
            const auto& distributors = NRealTime::ParseDistributorString(servers);
            Copy(distributors.begin(), distributors.end(), back_inserter(replica));

            if (Resolvable(replica, required, preferred)) {
                result.push_back(replica);
                resolved++;
            } else {
                FATAL_LOG << "Skipping unresolvable replica " << int(replica.Id) << Endl;
                unresolved++;
            }
        }
    }

    if (resolved + unresolved) {
        const float fraction = float(unresolved) / (resolved + unresolved);
        CHECK_WITH_LOG(fraction >= 0);
        VERIFY_WITH_LOG(fraction <= stream.MaxUnresolvedReplicas, "Critical number of unresolved replicas reached");
        if (fraction > stream.WarnUnresolvedReplicas) {
            ERROR_LOG << "Too many unresolved distributor replicas."
                         " Current part of unresolved replicas(" << fraction << ")"
                         " is greater than warning threshold(" << stream.WarnUnresolvedReplicas << ")" << Endl;
        }
    }

    return result;
}

void TDocFetcherConfig::PrintClientReplicas(const TConsistentClientReplicas& replicas, IOutputStream& so) const {
    for (auto&& replica : replicas) {
        so << "<Replica>" << Endl;
        so << "Servers : " << NRealTime::SerializeDistributorString(replica) << Endl;
        so << "Id : " << int(replica.Id) << Endl;
        so << "Priority : " << replica.Priority << Endl;
        so << "</Replica>" << Endl;
    }
}


TPersQueueStreamConfig::TReplicaIdsList TDocFetcherConfig::ParseReplicaIds(const TYandexConfig::TSectionsMap& sections) const {
    TPersQueueStreamConfig::TReplicaIdsList result;

    for (auto&& section : sections) {
        if (section.first == "ReplicaIds") {
            const auto& directives = section.second->GetDirectives();
            TPersQueueStreamConfig::TReplicaIds replicaIds;
            VERIFY_WITH_LOG(directives.GetValue("Id", replicaIds.first), "Missing ReplicaIds.Id directive");
            VERIFY_WITH_LOG(directives.GetValue("Replicas", replicaIds.second), "Missing ReplicaIds.Replicas directive");
            result.push_back(replicaIds);
        }
    }

    return result;
}
void TDocFetcherConfig::PrintReplicaIds(const TPersQueueStreamConfig::TReplicaIdsList& replicaIdsList, IOutputStream& so) const {
    for (auto&& replicaIds : replicaIdsList) {
        so << "<ReplicaIds>" << Endl;
        so << "Id : " << replicaIds.first << Endl;
        so << "Replicas : " << replicaIds.second << Endl;
        so << "</ReplicaIds>" << Endl;
    }
}

TConsistentClientOptions TDocFetcherConfig::ParseClientOptions(const TString& value) const {
    TConsistentClientOptionsParser parser(value);
    return parser;
}

TString TDocFetcherConfig::PrintClientOptions(const TConsistentClientOptions& value) const {
    return ::PrintConsistentClientOptions(value);
}

NRealTime::TDistributors TDocFetcherConfig::PreresolveAndFilter(const NRealTime::TDistributors& distributors) const {
    auto required = NBus::EIP_VERSION_ANY;
    auto preferred = NBus::EIP_VERSION_6;

    NRealTime::TDistributors result;
    for (auto d : distributors) {
        if (Resolvable(d, required, preferred)) {
            result.push_back(d);
        } else {
            FATAL_LOG << "Skipping unresolvable " << d << Endl;
        }
    }

    return result;
}
}

IDaemonModuleConfig::TFactory::TRegistrator<NFusion::TDocFetcherConfig> DocfetcherConfigRegistrator(TString{NFusion::DocfetcherModuleName});
