#include "connection.h"
#include "master_main.h"
#include "master_process.h"

#include <balancer/kernel/balancing/updater.h>
#include <balancer/kernel/connection_manager/connection_manager.h>
#include <balancer/kernel/client_request/backend.h>
#include <balancer/kernel/ctl/children_process_common.h_serialized.h>
#include <balancer/kernel/helpers/syscalls.h>
#include <balancer/kernel/module/module_requirements.h>
#include <balancer/kernel/process/resolve_helper.h>
#include <balancer/kernel/process/sd/sd.h>

#include <library/cpp/digest/md5/md5.h>

#include <sys/resource.h>


using namespace NConfig;
using namespace NSrvKernel;

namespace {
    constexpr TStringBuf UNIFIED_AGENT_LOG_NAME_PREFIX = "unified_agent:";
    constexpr size_t MaxExtraChildrenProcesses = GetEnumItemsCount<NSrvKernel::NProcessCore::TChildProcessType>();

    void ReadIgnoreBindErrors(THashSet<TIpAddr>& ibe, const TString& file) {
        if (!TFsPath(file).Exists()) {
            return;
        }
        TFileInput fin(file);
        TString addr;
        while (fin.ReadLine(addr)) {
            if (addr = StripInPlace(addr)) {
                Y_TRY(TError, err) {
                    TIpAddr res;
                    Y_PROPAGATE_ERROR(TIpAddr::FromIp(addr).AssignTo(res));
                    ibe.insert(res);
                    return {};
                } Y_CATCH {
                    Y_LOG_STDERR("WARN")
                        << "bad ip " << addr << " in " << file << ": " << GetErrorMessage(err) << Endl;
                }
            }
        }
    }

    struct TEnv {
        TEnv(size_t workers, std::function<void(IWorkerCtl*)> function)
            : Progress(0ul)
            , Workers(workers)
            , Function(std::move(function))
            , Channel(1ul)
        {}

        size_t Progress;
        const size_t Workers;
        TMutex Mutex;
        TVector<TError> Errors;
        std::function<void(IWorkerCtl*)> Function;
        TW2UChannel<char> Channel;
    };

    std::function<void(IWorkerCtl*)> GetFunctionWrapper(TAtomicSharedPtr<TEnv> env) {
        auto lambda = [env(std::move(env))](NSrvKernel::IWorkerCtl* process) {
            TError error{};
            try {
                env->Function(process);
            } catch (const yexception& exc) {
                error = MakeError<yexception>(exc);
            } catch (...) {
                error = MakeError<yexception>(yexception{} << CurrentExceptionMessage());
            }

            with_lock(env->Mutex) {
                if (error) {
                    env->Errors.push_back(std::move(error));
                }
                if (++env->Progress == env->Workers) {
                    Y_UNUSED(env->Channel.TrySend('a'));
                }
            }
        };

        return lambda;
    }

    TError ValidateExecution(const TVector<bool>& results, TVector<TError>& errors) {
        // find problem with delivery
        for (const auto& i : results) {
            if (!i) {
                return Y_MAKE_ERROR(yexception() << "channel couldn't send message");
            }
        }

        // find problem with execution
        for (auto& i : errors) {
            Y_PROPAGATE_ERROR(std::move(i));
        }

        return {};
    }
}

namespace NSrvKernel::NProcessCore {

    class TConfigCheckError : public yexception {};

    using TAcceptFull = TContListener::ICallBack::TAcceptFull;

    static const TVector<ui64> PingTimeIntervals = {
        0,
        10000,
        15000,
        25000,
        50000,
        75000,
        100000,
        125000,
        150000,
        175000,
        200000,
        250000,
        500000,
        750000,
        1000000,
        2500000,
        5000000,
        7500000,
        10000000,
        25000000
    };

    THolder<TOwnExecutor> CreateExecutor(
        const TMainOptions& options,
        const TMainConfig& config,
        NCoro::IEnterPollerCallback* enterPollerCallback,
        NCoro::IScheduleCallback* scheduleCallback
    ) noexcept {
        auto executor = MakeHolder<TOwnExecutor>(
            options.CoroStackSize(),
            options.Poller,
            options.CoroStackGuard,
            options.CoroPoolSettings,
            enterPollerCallback,
            scheduleCallback
        );
        executor->Executor().SetFailOnError(config.CoroFailOnError);
        return executor;
    }

    TError SetNoFileLimitAndValidate(rlim_t softLimit) noexcept {
        struct rlimit rlim = {};
        Y_PROPAGATE_ERROR(Y_SYSCALL(getrlimit(RLIMIT_NOFILE, &rlim)));
        rlim_t hardLimit = rlim.rlim_max;
        Y_REQUIRE(softLimit < hardLimit, yexception() << "not enough file descriptors");
        if (rlim.rlim_cur >= hardLimit) {
            return {};
        }
        rlim.rlim_cur = hardLimit;
        Y_PROPAGATE_ERROR(Y_SYSCALL(setrlimit(RLIMIT_NOFILE, &rlim)));
        Y_PROPAGATE_ERROR(Y_SYSCALL(getrlimit(RLIMIT_NOFILE, &rlim)));
        Y_REQUIRE(hardLimit == rlim.rlim_cur, yexception() << "nofile limit was not set");
        return {};
    }

    TMainTask::TMainTask(
        const TString& config,
        TGlobals& globals,
        IModuleFactory* loader,
        TMainOptions& options,
        TTwoNonblockingPipes* initializerPipes
    )
        : InitializerPipes_(initializerPipes)
        , Loader_(loader)
        , SharedStatsManager_(SharedAllocator_)
        , ProtocolFactory_(MakeHolder<TBackendProtocolFactory>())
        , MainOptions_(options)
        , ModuleRequirements_(MakeHolder<TModuleRequirements>())
    {
        TPrintOnceScope printOnceScope;

        ConfigHash_ = MD5::Data(config);

        Config_.WorkerStartDelay = options.WorkerStartDelay;

        Config_.ConfigString = config;
        TStringInput cfgin(Config_.ConfigString);

        auto configParser = ConfigParser(cfgin, globals);

        HandleResolveOnStart(configParser.Get());

        InitHeadersParser();

        ParseInstanceParams(configParser.Get());

        if (Config_.CoroStackSize.Defined()) {
            // Try - because command line argument has higher priority
            options.TrySetCoroStackSize(*Config_.CoroStackSize);
        }
        options.CoroStackGuard = Config_.CoroStackGuard;
        if (Config_.CoroPoolAllocator) {
            options.CoroPoolSettings = Config_.CoroPoolSettings;
        }

        ChannelListener = MakeHolder<TChannelListener>(Config_.NChildren);

        if (IsExtendedConfigCheck()) {
            Config_.StartWithoutSpecialChildren = true;
            Config_.NChildren = 1;
        }

        // ConnectionManager должен быть создан после того, как GetCountOfChildren начнёт возвращать окончательное число воркеров
        ParseMap(configParser.Get(), [this](const auto& key, auto* value) {
            if (key == "connection_manager") {
                if (ConnectionManager_) {
                    ythrow NConfig::TConfigParseError() << "try to configure connection manager twice";
                }
                ConnectionManager_ = MakeIntrusive<NConnectionManager::TConnectionManager>(*this, *value->AsSubConfig());
                Config_.ChildrenMask |= TChildProcessType::ConnectionManager;
                return;
            }
        });

        if (Config_.SetNoFile) {
            Y_TRY(TError, err) {
                return SetNoFileLimitAndValidate(2 * Config_.NChildren * Max(10000ul, 5000ul + Config_.MaxConnections));
            } Y_CATCH {
                err.Throw(); // let it die
            }
        }

        // after this call SharedStatsManager_ is ready to register new signals
        SharedStatsManager_.SetWorkersCount(Config().NChildren, MaxExtraChildrenProcesses);

        MainStats_ = MakeHolder<TMainStats>(SharedStatsManager_);
        ProcessStatOwner_ = MakeHolder<TProcessStatOwner>(SharedStatsManager_, GetCountOfChildren());
        MakePingerStatsCounters();

        ParseMap(configParser.Get(), [this](const auto& key, auto* value) {
            if (key == "sd") {
                if (SDManager_) {
                    ythrow NConfig::TConfigParseError() << "try to configure service discovery twice";
                }

                SDManager_ = MakeHolder<TSDManager>(*value->AsSubConfig(), SharedStatsManager_);
                return;
            }
        });

        UpdaterManager_ = MakeHolder<NDynamicBalancing::TUpdaterManager>(
            SharedStatsManager_,
            Config().BackendsBlacklistFile,
            PingerStatsCounters_.Get()
        );

        BannedAddresses_ = MakeHolder<TBannedAddresses>(SharedStatsManager_);

        // pre-parse addresses to fill disabled addresses
        ParseMap(configParser.Get(), [&](const auto& key, auto* value) {
            // if we have this field, we parse balancer config, not server or client
            if (key == "addrs") {
                if (!Trees.empty()) {
                    ythrow TConfigParseError() << "try to parse addresses twice";
                }
                Trees.AddTree();
                value->AsSubConfig()->ForEach(&Trees.front().ListenAddrs);
                return;
            }
        });

        if (Config_.Dns.ResolveCachedIpIfNotSet) {
            Y_VERIFY(ResolveHelper_);
            ResolveHelper_->WaitForResults();
        }
        Y_ASSERT(Config_.Dns.ResolveCachedIpIfNotSet == !!ResolveHelper_);

        configParser->ForEach(this);

        ApplyModuleRequirements();

        HandleChildren();

        CheckConfigParams();

        if (options.JustDumpSignals) {
            SharedStatsManager_.DumpSignals();
        }
    }

    TMainTask::~TMainTask() {
#ifdef _linux_
        // Potentially tc-bpf program could stay alive
        // in the kernel after program termination.

        if (Config_.P0fEnabled) {
            NP0f::UnloadBpfProgram();
        }
#endif
        Dispose();
    }

    void TMainTask::Execute(NThreading::TPromise<void>* startPromise) {
        if (IsBasicConfigCheck()) {
            return;
        }

        if (SDManager_) {
            SDManager_->Start();
        }

        TMasterProcess masterProcess(this);

        SharedFiles_.Start();

        MainStats_->TcpMaxConns.Set(Config_.NChildren * Config_.MaxConnections);
        MainStats_->MappedSharedMemory.Set(SharedAllocator_.MappedBytes());
        MainStats_->AllocatedSharedMemory.Set(SharedAllocator_.AllocatedBytes());

        masterProcess.Executor().CreateOwned([&](TCont*) {
            masterProcess.Execute(startPromise);
        }, "TMasterProcess::Execute");
        masterProcess.Executor().Execute();
        Y_ENSURE_EX(!Failed_, TConfigCheckError());
    }

    void TMainTask::StopMaster(const TGracefulShutdownOpts& opts) {
        Y_UNUSED(StopChannel_.TrySend(opts));
    }

    void TMainTask::DoDispose() noexcept {
        // Dispose all log writing threads
        Log_.Dispose();
        PingerLog_.Dispose();
        StorageGCLog_.Dispose();
        ConnectionManagerLog_.Dispose();
        for (auto& i : Logs_) {
            i.second->Dispose();
        }
    }

    TLog* TMainTask::CreateLog(const TString& name) {
        auto& log = Logs_[name];

        if (!log) {
            if (name.empty()) {
                log = MakeAtomicShared<TContextSafeLogGenerator<TLog>>(MakeHolder<TStreamLogBackend>(&Cerr));
            } else if (name.StartsWith(UNIFIED_AGENT_LOG_NAME_PREFIX)) {
                auto counters = SharedStatsManager_.GetUnifiedAgentCounters(name);
                // Default output stream is stdout(if SetLog is not called).
                // Replace it with stderr if file-based backend is not set.
                if (Config_.UnifiedAgentLogName.empty()) {
                    UnifiedAgentLog_.Reset(new TLog(MakeHolder<TStreamLogBackend>(&Cerr)));
                } else {
                    UnifiedAgentLog_.Reset(new TLog(MakeHolder<TFileLogBackend>(Config_.UnifiedAgentLogName)));
                }
                auto clientParameters = NUnifiedAgent::TClientParameters(name.substr(
                    UNIFIED_AGENT_LOG_NAME_PREFIX.length(),
                    name.length() - UNIFIED_AGENT_LOG_NAME_PREFIX.length()
                ));
                clientParameters.SetLog(*UnifiedAgentLog_.Get());
                clientParameters.SetCounters(counters);
                log = MakeAtomicShared<TContextSafeLogGenerator<TLog>>(NUnifiedAgent::MakeLogBackend(clientParameters));
            } else {
                log = MakeAtomicShared<TContextSafeLogGenerator<TLog>>(MakeHolder<TFileLogBackend>(name));
            }
        }

        return log.Get();
    }

    TSharedAllocator& TMainTask::SharedAllocator() noexcept {
        return SharedAllocator_;
    }

    TSharedStatsManager& TMainTask::SharedStatsManager() noexcept {
        return SharedStatsManager_;
    }

    bool TMainTask::IsPingerEnabled() const noexcept {
        return Config_.ChildrenMask & TChildProcessType::Pinger;
    }

    NDynamicBalancing::TUpdaterManager& TMainTask::DynamicBalancingUpdaterManager() noexcept {
        return *UpdaterManager_;
    }

    NConnectionManager::TConnectionManager* TMainTask::ConnectionManager() noexcept {
        return ConnectionManager_.Get();
    }

    THolder<IModuleData>& TMainTask::GetModuleData(TStringBuf name) noexcept {
        return ModuleData_[name];
    }

    NYP::NServiceDiscovery::TEndpointSetManager* TMainTask::GetSDManager() noexcept {
        return SDManager_.Get();
    }

    TBackendProtocolFactory& TMainTask::GetProtocolFactory() const noexcept {
        return *ProtocolFactory_;
    }

    // instead of START_PARSE, in order to make it work in a bloody cpp file
    void TMainTask::DoConsume(const TString &key, NConfig::IConfig::IValue *value) {
        if (key == "admin_addrs") {
            value->AsSubConfig()->ForEach(&Config_.AdminAddrs);
            return;
        }

        ON_KEY("log", Config_.LogName) {
            Log_.Open(Config_.LogName);
            return;
        }

        TString pingerLogName;
        ON_KEY("pinger_log", pingerLogName) {
            PingerLog_.Open(pingerLogName);
            return;
        }

        TString connectionManagerLogName;
        ON_KEY("connection_manager_log", connectionManagerLogName) {
            ConnectionManagerLog_.Open(connectionManagerLogName);
            return;
        }

        TString storageLogName;
        ON_KEY("storage_log", storageLogName) {
            StorageGCLog_.Open(storageLogName);
            return;
        }
        // preparsed before
        if (IsIn({
            "coro_stack_size",
            "coro_stack_guard",
            "coro_pool_allocator",
            "coro_pool_rss_keep",
            "coro_pool_small_rss_keep",
            "coro_pool_release_rate",
            "coro_pool_stacks_per_chunk",
            "addrs",
            "unistat",
            "pinger_state_directory",
            "state_directory",
            "thread_mode",
            "watchdog",
            "set_no_file",
            "backends_blacklist",
            "dynamic_balancing_log",
            "workers",
            "events",
            "enable_reuse_port",
            "deferaccept",
            "config_check",
            "tcp_fastopen",
            "unified_agent_log",
            "pinger_required",
            "updater_required",
            "storage_gc_required",
            "config_uid",
            "connection_manager"
        }, key)) {
            return;
        }

        ON_KEY("ban_addresses_disable_file", Config_.BanAddressesDisableFile_) {
            return;
        }

        ON_KEY("ban_addresses_max_count", Config_.BanAddressesMaxCount_) {
            return;
        }

        if (key == "cpu_limiter") {
            TCpuLimiterBuilder cpuLimiterBuilder;
            cpuLimiterBuilder.ParseConfig(value->AsSubConfig());
            CpuLimiterConfig_ = MakeHolder<TCpuLimiterConfig>(cpuLimiterBuilder.GetConfig());
            CpuLimiterStat_ = MakeHolder<TCpuLimiterStat>(SharedStatsManager_);
            return;
        }

        if (key == "sd") {
            if (!SDManager_) {
                ythrow NConfig::TConfigParseError() << "something wrong with service discovery config, should be preparsed and already created";
            }
            return;
        }

        ON_KEY("maxconn", Config_.MaxConnections) {
            return;
        }

        ON_KEY("sosndbuf", Config_.SoSndbuf) {
            return;
        }

        ON_KEY("temp_buf_prealloc", Config_.TempBufPrealloc) {
            return;
        }

        ON_KEY("config_tag", Config_.ConfigTag) {
            return;
        }

        ON_KEY("worker_start_delay", Config_.WorkerStartDelay) {
            return;
        }

        ON_KEY("worker_start_duration", Config_.WorkerStartDuration) {
            return;
        }

        ON_KEY("buffer", Config_.Buffer) {
            return;
        }

        if (key == "stats_connect_timeout") {
            return;
        }

        if (key == "stats_timeout") {
            return;
        }

        if (key == "stats_event_timeout") {
            return;
        }

        ON_KEY("tcp_keep_cnt", Config_.TcpKeepalive.Cnt) {
            return;
        }

        ON_KEY("tcp_keep_idle", Config_.TcpKeepalive.Idle) {
            return;
        }

        ON_KEY("tcp_keep_intvl", Config_.TcpKeepalive.Intvl) {
            return;
        }

        ON_KEY("tcp_listen_queue", Config_.TcpListenQueue) {
            return;
        }

        {
            TString file;
            ON_KEY("ignore_bind_errors_file", file) {
                ReadIgnoreBindErrors(IgnoreBindErrors_, file);
                return;
            }
        }

        ON_KEY("tcp_congestion_control", Config_.TcpCongestionControlName) {
            Y_TRY(TError, err) {
                return ValidateCongestionControl(Config_.TcpCongestionControlName);
            } Y_CATCH {
                Y_WARN_ONCE("cannot use tcp_congestion_control " << Config_.TcpCongestionControlName.Quote() << ": " << GetErrorMessage(err));
                Config_.TcpCongestionControlName.clear();
            }
            return;
        }

        int32_t tcpNotsentLowat = 0;
        ON_KEY("tcp_notsent_lowat", tcpNotsentLowat) {
            Config_.TcpNotsentLowat = tcpNotsentLowat;
            return;
        }

        ON_KEY("default_tcp_rst_on_error", Config_.TcpRstOnError) {
            return;
        }

        ON_KEY("shutdown_accept_connections", Config_.ShutdownAcceptConnections) {
            return;
        }

        ON_KEY("shutdown_close_using_bpf", Config_.ShutdownCloseUsingBPF) {
            return;
        }

        if (key == "disable_xml_stats") {
            return;
        }

        ON_KEY("log_queue_max_size", Config_.LogQueueOpts.MaxQueueSize) {
            if (Config_.LogQueueOpts.MaxQueueSize <= 0) {
                ythrow TConfigParseError() << "log_queue_max_size should be a positive integer";
            }
            return;
        }

        ON_KEY("log_queue_submit_attempts_count", Config_.LogQueueOpts.SubmitAttemptsCount) {
            if (Config_.LogQueueOpts.SubmitAttemptsCount <= 0) {
                ythrow TConfigParseError() << "log_queue_submit_attempts_count should be a positive integer";
            }
            return;
        }

        ON_KEY("log_queue_flush_interval", Config_.LogQueueOpts.FlushInterval) {
            return;
        }

        ON_KEY("_sock_inbufsize", Config_.SockBufSize.Rcv) {
            return;
        }

        ON_KEY("_sock_outbufsize", Config_.SockBufSize.Snd) {
            return;
        }

        ON_KEY("_coro_fail_on_error", Config_.CoroFailOnError) {
            return;
        }

        ON_KEY("dns_timeout", Config_.Dns.Timeout) {
            return;
        }

        ON_KEY("dns_ttl", Config_.Dns.Ttl) {
            return;
        }

        ON_KEY("dns_error_ttl", Config_.Dns.ErrorTtl) {
            return;
        }

        ON_KEY("dns_async_resolve", Config_.Dns.AsyncResolve) {
            return;
        }

        ON_KEY("dns_resolve_cached_ip_if_not_set", Config_.Dns.ResolveCachedIpIfNotSet) {
            return;
        }

        ON_KEY("dns_resolve_on_start_timeout", Config_.Dns.ResolveOnStart) {
            return;
        }

        ON_KEY("dns_ip", Config_.Dns.Ip) {
            return;
        }

        ON_KEY("dns_port", Config_.Dns.Port) {
            return;
        }

        ON_KEY("dns_its_switch_file", Config_.Dns.SwitchFile) {
            return;
        }

        ON_KEY("dns_its_switch_check", Config_.Dns.SwitchCheckTimeout) {
            return;
        }

        ON_KEY("reset_dns_cache_file", Config_.Dns.ResetCacheFile) {
            return;
        }

        if (key == "private_address") {
            return;
        }

        if (key == "build_info") {
            return;
        }

        ON_KEY("p0f_enabled", Config_.P0fEnabled) {
            return;
        }

        ON_KEY("p0f_map_max_size", Config_.P0fMapMaxSize) {
            return;
        }

        {
            if (Trees.empty()) {
                ythrow TConfigParseError() << "try parse module, but listening addresses are empty";
            }

            TTree* mainTree = dynamic_cast<TTree*>(&Trees.front().Get());

            if (!mainTree) {
                ythrow TConfigParseError() << "no suitable module tree to build";
            }

            if (mainTree->Entry) {
                ythrow TConfigParseError() << "two modules in instance: " << key << " and " << mainTree->Entry->GetHandle()->Name();
            }

            TModuleParams::TDisabledHosts disabledHosts;

            const auto& disabledAddrs = Trees.front().ListenAddrs.DisabledHosts();
            for (const auto& addr: disabledAddrs) {
                disabledHosts.insert(addr);
            }
            Trees.front().ListenAddrs.ClearDisabledHosts();

            const TModuleParams mp {
                value->AsSubConfig()
                , Loader_
                , this
                , &mainTree->Modules
                , ResolveHelper_ ? ResolveHelper_->GetHosts() : nullptr
                , &disabledHosts
            };

            mainTree->Entry = Loader_->MustLoad(key, mp);
            mainTree->Modules.Finalize();

            return;
        }
    }


    void RunMasterMain(TGlobals& globals, const TString& config, IModuleFactory* loader, TMainOptions& options, TTwoNonblockingPipes* initializerPipes) {
        NProcessCore::TMainTask mainTask(config, globals, loader, options, initializerPipes);
        mainTask.Execute();
    }

    void TMainTask::MakePingerStatsCounters() noexcept {
        PingerStatsCounters_ = MakeHolder<TPingerStatsCounters>(
            SharedStatsManager_.MakeCounter("total_pings").AllowDuplicate().Build(),
            SharedStatsManager_.MakeCounter("successful_pings").AllowDuplicate().Build(),
            SharedStatsManager_.MakeCounter("failed_pings").AllowDuplicate().Build(),
            SharedStatsManager_.MakeCounter("bad_status_pings").AllowDuplicate().Build(),
            SharedStatsManager_.MakeCounter("zero_weight_pings").AllowDuplicate().Build(),
            SharedStatsManager_.MakeCounter("parse_failed_pings").AllowDuplicate().Build(),
            SharedStatsManager_.MakeCounter("connection_refused_pings").AllowDuplicate().Build(),
            SharedStatsManager_.MakeCounter("connection_timeout_pings").AllowDuplicate().Build(),
            SharedStatsManager_.MakeCounter("connection_other_error_pings").AllowDuplicate().Build(),
            SharedStatsManager_.MakeHistogram("ping_processing_time", PingTimeIntervals).AllowDuplicate().Scale(1e6).Build(),
            SharedStatsManager_.MakeCounter("degraded_pings").AllowDuplicate().Build()
        );
    }

    void TMainTask::InitLogWritingThreads(TContExecutor& executor, size_t workerId) noexcept {
        Log_.CreateWritingThreadAndCommonStuff(executor, Config_.LogQueueOpts, *SystemThreadFactory(), MainStats_->ProcessedLogItems, MainStats_->LostLogItems, workerId);
        PingerLog_.CreateWritingThreadAndCommonStuff(executor, Config_.LogQueueOpts, *SystemThreadFactory(), MainStats_->ProcessedLogItems, MainStats_->LostLogItems, workerId);
        StorageGCLog_.CreateWritingThreadAndCommonStuff(executor, Config_.LogQueueOpts, *SystemThreadFactory(), MainStats_->ProcessedLogItems, MainStats_->LostLogItems, workerId);
        ConnectionManagerLog_.CreateWritingThreadAndCommonStuff(executor, Config_.LogQueueOpts, *SystemThreadFactory(), MainStats_->ProcessedLogItems, MainStats_->LostLogItems, workerId);
        DynamicBalancingLog_.CreateWritingThreadAndCommonStuff(executor, Config_.LogQueueOpts, *SystemThreadFactory(), MainStats_->ProcessedLogItems, MainStats_->LostLogItems, workerId);
        for (auto& log : Logs_) {
            log.second->CreateWritingThreadAndCommonStuff(executor, Config_.LogQueueOpts, *SystemThreadFactory(), MainStats_->ProcessedLogItems, MainStats_->LostLogItems, workerId);
        }
    }

    void TMainTask::ShutdownWithError(TStringBuf message) noexcept {
        Y_LOG_STDERR("FAIL") << message << Endl;
        Failed_ = true;
        StopMaster({});
    }

    size_t TMainTask::GetCountOfChildren() const noexcept {
        return Config_.NChildren + MaxExtraChildrenProcesses;
    }

    NRpsLimiter::TQuotaManager* TMainTask::GetQuotaManager() noexcept {
        return QuotaManager_.Get();
    }

    void TMainTask::AddModules(const TString& config, const TGlobals& globals, TNodeFactory<IModule>& factory,
                               TAtomic* totalConnectionInProgress) {
        TStringInput cfgin(config);

        auto tree = MakeHolder<TTree>();
        tree->TotalConnectionInProgress = totalConnectionInProgress;
        TConfigAddresses addresses;

        auto configParser = ConfigParser(cfgin, globals);

        // pre-parse addresses to fill disabled addresses
        ParseMap(configParser.Get(), [&](const auto& key, auto* value) {
            if (key == "addrs") {
                value->AsSubConfig()->ForEach(&addresses);
                return;
            }
        });

        TModuleParams::TDisabledHosts disabledHosts;

        const auto& disabledAddrs = addresses.DisabledHosts();
        for (const auto& addr: disabledAddrs) {
            disabledHosts.insert(addr);
        }
        addresses.ClearDisabledHosts();

        ParseMap(configParser.Get(), [&](const auto& key, auto* value) {
            if (key == "addrs") {
                return;
            }

            {
                const TModuleParams mp{
                    value->AsSubConfig()
                    , &factory
                    , this
                    , &tree->Modules
                    , ResolveHelper_ ? ResolveHelper_->GetHosts() : nullptr
                    , &disabledHosts
                };

                tree->Entry = factory.MustLoad(key, mp);
                tree->Modules.Finalize();

                return;
            }
        });

        if (HasConflictingAddresses(addresses, SharedStatsManager_.Addrs())) {
            ythrow TConfigParseError{} << "listening addresses conflict with unistat addresses";
        }

        if (HasConflictingAddresses(addresses, Config().AdminAddrs)) {
            ythrow TConfigParseError{} << "listening addresses conflict with admin addresses";
        }

        for (const auto& i : Trees) {
            if (HasConflictingAddresses(addresses, i.ListenAddrs)) {
                ythrow TConfigParseError{} << "listening addresses of different module trees conflict";
            }
        }

        Trees.AddTree(std::move(addresses), std::move(tree));
    }

    THolder<TTree> TMainTask::CreateTree(const TString& config, const TString& name, TNodeFactory<IModule>& factory) {
        TStringInput configInput(config);
        auto configParser = ConfigParser(configInput);

        auto tree = MakeHolder<TTree>();

        const TModuleParams mp {
            configParser.Get(),
            &factory,
            this,
            &tree->Modules,
            ResolveHelper_ ? ResolveHelper_->GetHosts() : nullptr
        };

        tree->Entry = factory.MustLoad(name, mp);
        tree->Modules.Finalize();

        return tree;
    }

    void TMainTask::AddExternalTree(ITree& tree) {
        TUniversalGuard guard(TreesMutex);
        if (!guard.Lock()) {
            ythrow yexception{} << "can't add external tree";
        }

        Trees.AddTree(tree);

        auto init = [&tree](NSrvKernel::IWorkerCtl* process) {
            tree.Init(process);
        };

        if (AtomicGet(LiveWorkersCounter_)) {
            TryRethrowError(ExecuteFunctionByWorkersAndWait(init));
        }
    }

    void TMainTask::RemoveExternalTree(ITree& tree) {
        TUniversalGuard guard(TreesMutex);
        if (!guard.Lock()) {
            ythrow yexception{} << "can't remove external tree";
        }

        auto dispose = [&tree](NSrvKernel::IWorkerCtl* process) {
            tree.Dispose(process);
        };

        if (AtomicGet(LiveWorkersCounter_)) {
            TryRethrowError(ExecuteFunctionByWorkersAndWait(dispose));
        }

        Trees.RemoveTree(tree);
    }

    void TMainTask::TTrees::Init(IWorkerCtl* process) {
        for (auto& tree : *this) {
            tree.Get().Init(process);
        }
    }

    void TMainTask::TTrees::Dispose(IWorkerCtl* process) {
        for (auto& tree : *this) {
            tree.Get().Dispose(process);
        }
    }

    void TMainTask::TTrees::ShowListen(TEventData& event) const {
        IOutputStream& out = event.RawOut();
        out << "<addrs>";
        for (const auto& tree : *this) {
            for (const auto& listenAddr : tree.ListenAddrs) {
                for (auto addr = listenAddr.Begin(); addr != listenAddr.End(); ++addr) {
                    out << "<addr>" << PrintHostAndPort(NAddr::TAddrInfo(&*addr)) << "</addr>";
                }
            }
        }
        out << "</addrs>";
    }

    TSharedFiles& TMainTask::GetSharedFiles() noexcept {
        return SharedFiles_;
    }

    void TMainTask::CloseListeners(const TGracefulShutdownOpts& opts) {
        auto func = [&opts](NSrvKernel::IWorkerCtl* process) {
            process->CloseListeners(opts);
        };

        TryRethrowError(ExecuteFunctionByWorkersAndWait(func));
    }

    TError TMainTask::ExecuteFunctionByWorkersAndWait(std::function<void(IWorkerCtl*)> func) {
        TAtomicSharedPtr<TEnv> env = MakeAtomicShared<TEnv>(ChildrenChannels_.size(), std::move(func));

        auto lambda = GetFunctionWrapper(env);

        TVector<bool> results(Reserve(ChildrenChannels_.size()));
        for (auto& i : ChildrenChannels_) {
            results.push_back(i->Send(lambda, TInstant::Max()) == EChannelStatus::Success);
        }

        char c;
        EChannelStatus status = env->Channel.Receive(c, TInstant::Max());
        switch (status) {
            case EChannelStatus::Canceled:
                return Y_MAKE_ERROR(TSystemError(ECANCELED));
            case EChannelStatus::TimedOut:
                return Y_MAKE_ERROR(TSystemError(ETIMEDOUT));
            case EChannelStatus::Success:
                break;
        }

        TGuard<TMutex> guard(env->Mutex);

        Y_PROPAGATE_ERROR(ValidateExecution(results, env->Errors));

        return {};
    }

    TChannelListener& TMainTask::GetChannelListener() {
        return *ChannelListener;
    }

    TProcessStatOwner& TMainTask::GetProcessStatOwner() const {
        Y_VERIFY(ProcessStatOwner_, "ProcessStatOwner has not been created yet");
        return *ProcessStatOwner_;
    }

    void TMainTask::RegisterStorage(NStorage::TStorage* st) {
        Storages_.push_back(st);
    }

    TModuleRequirements& TMainTask::GetModuleRequirements() const noexcept {
        return *ModuleRequirements_;
    }

    void TMainTask::SetupSd(const NYP::NServiceDiscovery::TSDConfig& config) {
        if (SDManager_) {
            ythrow yexception{} << "service discovery already initialized";
        }
        SDManager_ = MakeHolder<TSDManager>(config, SharedStatsManager_);
    }

    bool TMainTask::ReopenAllLogs(IOutputStream& out) noexcept {
        bool withoutErrors = true;

        for (auto& log: Logs_) {
            try {
                log.second->ReopenLog();
            } catch (const yexception& e) {
                withoutErrors = false;
                out << "reopening " << log.first << " failed: " << e.what() << '\n';
            }
        }

        try {
            Log_.ReopenLog();
        } catch (const yexception& e) {
            withoutErrors = false;
            out << "reopening " << Config_.LogName << " failed: " << e.what() << '\n';
        }

        try {
            DynamicBalancingLog_.ReopenLog();
        } catch (...) {
            withoutErrors = false;
            out << "reopening dynamic balancing log failed: " << CurrentExceptionMessage() << '\n';
        }

        try {
            PingerLog_.ReopenLog();
        } catch (...) {
            withoutErrors = false;
            out << "reopening pinger log failed: " << CurrentExceptionMessage() << '\n';
        }

        try {
            StorageGCLog_.ReopenLog();
        } catch (...) {
            withoutErrors = false;
            out << "reopening storage log failed: " << CurrentExceptionMessage() << '\n';
        }

        try {
            ConnectionManagerLog_.ReopenLog();
        } catch (...) {
            withoutErrors = false;
            out << "reopening connection manager log failed: " << CurrentExceptionMessage() << '\n';
        }

        try {
            if (SDManager_) {
                SDManager_->ReopenLog();
            }
        } catch (...) {
            withoutErrors = false;
            out << "reopening service discovery log failed: " << CurrentExceptionMessage() << '\n';
        }

        out << "Reopen log complete.";
        return withoutErrors;
    }

    void TMainTask::ReopenAllLogsFromThread() noexcept {
        Y_UNUSED(ReopenLogChannel_.TrySend('a'));
    }

    void TMainTask::HandleResolveOnStart(NConfig::IConfig* configParser) {
        ParseMap(configParser, [&](const auto& key, auto* value) {
            ON_KEY("dns_resolve_cached_ip_if_not_set", Config_.Dns.ResolveCachedIpIfNotSet) {
                return;
            }
            ON_KEY("dns_resolve_on_start_timeout", Config_.Dns.ResolveOnStart) {
                return;
            }
        });
        if (Config_.Dns.ResolveCachedIpIfNotSet) {
            ResolveHelper_ = MakeHolder<TResolveHelper>(Config_.Dns.ResolveOnStart);
            ResolveHelper_->StartResolve(*configParser, AF_INET6);
        }
    }

    void TMainTask::ParseInstanceParams(NConfig::IConfig* configParser) {
        ParseMap(configParser, [&](const auto& key, auto* value) {
            size_t coroStackSize = 0;
            ON_KEY("coro_stack_size", coroStackSize) {
                Config_.CoroStackSize = coroStackSize;
                return;
            }
            ON_KEY("coro_stack_guard", Config_.CoroStackGuard) {
                return;
            }
            ON_KEY("coro_pool_allocator", Config_.CoroPoolAllocator) {
                return;
            }
            ON_KEY("coro_pool_rss_keep", Config_.CoroPoolSettings.RssPagesToKeep) {
                return;
            }
            ON_KEY("coro_pool_small_rss_keep", Config_.CoroPoolSettings.SmallStackRssPagesToKeep) {
                return;
            }
            ON_KEY("coro_pool_release_rate", Config_.CoroPoolSettings.ReleaseRate) {
                return;
            }
            ON_KEY("coro_pool_stacks_per_chunk", Config_.CoroPoolSettings.StacksPerChunk) {
                return;
            }

            ON_KEY("workers", Config_.NChildren) {
                return;
            }

            if (key == "rpslimiter_instance") {
                Config_.ChildrenMask |= TChildProcessType::QuotaSyncer;
                QuotaManager_ = NRpsLimiter::CreateQuotaManager(*value->AsSubConfig(), TInstant::Now());
                return;
            }

            if (key == "state_directory") {
                return;
            }

            if (key == "pinger_state_directory") {
                return;
            }

            bool pingerRequired = false;
            ON_KEY("pinger_required", pingerRequired) {
                if (pingerRequired) {
                    Config_.ChildrenMask |= TChildProcessType::Pinger;
                }
                return;
            }

            bool updaterRequired = false;
            ON_KEY("updater_required", updaterRequired) {
                if (updaterRequired) {
                    Config_.ChildrenMask |= TChildProcessType::Updater;
                }
                return;
            }

            bool storageGcRequired = false;
            ON_KEY("storage_gc_required", storageGcRequired) {
                if (storageGcRequired) {
                    Config_.ChildrenMask |= TChildProcessType::StorageGC;
                }
                return;
            }

            if (key == "thread_mode") {
                return;
            }

            ON_KEY("watchdog", Config_.Watchdog) {
                return;
            }

            ON_KEY("set_no_file", Config_.SetNoFile) {
                return;
            }

            ON_KEY("start_without_special_children", Config_.StartWithoutSpecialChildren) {
                return;
            }

            ON_KEY("backends_blacklist", Config_.BackendsBlacklistFile) {
                return;
            }

            ON_KEY("dynamic_balancing_log", Config_.DynamicBalancingLogName) {
                DynamicBalancingLog_.Open(Config_.DynamicBalancingLogName);
                return;
            }

            if (key == "unistat") {
                TSharedStatsManager::TConfig config;
                value->AsSubConfig()->ForEach(&config);
                if (config.MaxConns.Defined()) {
                    Config_.MaxStatsConnections = *config.MaxConns;
                }
                SharedStatsManager_.SetConfig(std::move(config));
                return;
            }

            ON_KEY("unified_agent_log", Config_.UnifiedAgentLogName) {
                return;
            }

            if (key == "config_check") {
                Y_ENSURE_EX(!ConfigCheck_, TConfigCheckError() << "config_check redefinition");
                ConfigCheck_.ConstructInPlace(this);
                value->AsSubConfig()->ForEach(&*ConfigCheck_);
                return;
            }
        });
    }

    void TMainTask::ApplyModuleRequirements() {
        if (!Config_.P0fEnabled && ModuleRequirements_->P0f) {
            Config_.P0fEnabled = true; // for AWACS - enable implicitly, because it is used in headers
        }
    }

    void TMainTask::HandleChildren() {
        if (Config_.NChildren) {
            Config_.ChildrenMask |= TChildProcessType::Default;
        }

        if (UpdaterManager_->RequiresWorkerStart()) {
            Config_.ChildrenMask |= TChildProcessType::Updater;
        }

        if (Config_.StartWithoutSpecialChildren) {
            Config_.ChildrenMask &= TChildProcessType::Default;
        }

        const size_t capacity = 10;
        ChildrenChannels_.reserve(GetCountOfChildren());
        for (size_t i = 0ul; i != Config_.NChildren; ++i) {
            ChildrenChannels_.emplace_back(MakeHolder<TU2WChannel<std::function<void(IWorkerCtl*)>>>(capacity));
        }

        if (Config_.ChildrenMask & TChildProcessType::Updater) {
            ChildrenChannels_.emplace_back(MakeHolder<TU2WChannel<std::function<void(IWorkerCtl*)>>>(capacity));
        }
    }

    void TMainTask::CheckConfigParams() {
        if (!Trees.empty()) {
            if (TTree* mainTree = dynamic_cast<TTree*>(&Trees.front().Get())) {
                // we parsed a self-sufficient config of balancer, not server or client
                if(!mainTree->Entry) {
                    ythrow TConfigParseError() << "listening addresses not empty, but no modules configured";
                }

                if (!Config_.NChildren) {
                    ythrow TConfigParseError() << "listening addresses not empty, but count of workers is zero";
                }

                if (HasConflictingAddresses(Trees.front().ListenAddrs, SharedStatsManager_.Addrs())) {
                    ythrow TConfigParseError{} << "listening addresses conflict with unistat addresses";
                }

                if (HasConflictingAddresses(Trees.front().ListenAddrs, Config().AdminAddrs)) {
                    ythrow TConfigParseError{} << "listening addresses conflict with admin addresses";
                }
            }
        }

        if (HasConflictingAddresses(SharedStatsManager_.Addrs(), Config().AdminAddrs)) {
            ythrow TConfigParseError{} << "unistat addresses conflict with admin addresses";
        }

        if (Config().AdminAddrs.empty() && !MainOptions_.AllowEmptyAdminAddrs) {
            ythrow TConfigParseError{} << "using multiple processes with no admin addrs";
        }

        if (Config().ShutdownAcceptConnections && Config().ShutdownCloseUsingBPF) {
            ythrow TConfigParseError() << "can't use 'shutdown_accept_connections' and 'shutdown_close_using_bpf' simultaneously";
        }
    }
}
