#include "merger_analyzer.h"

#include <saas/rtyserver/config/merger_config.h>
#include <saas/rtyserver/config/shards_config.h>
#include <saas/rtyserver/common/common_messages.h>
#include <saas/rtyserver/common/search_area_modifier.h>
#include <saas/rtyserver/common/sharding.h>
#include <saas/rtyserver/indexer_core/index_dir.h>
#include <saas/rtyserver/indexer_core/file_index_modify_guard.h>
#include <saas/rtyserver/indexer_core/messages.h>
#include <saas/rtyserver/index_storage/index_storage.h>
#include <saas/rtyserver/unistat_signals/signals.h>
#include <library/cpp/balloc/optional/operators.h>

#include <util/datetime/systime.h>
#include <util/folder/filelist.h>

using ESegmentsSort = NRTYServer::TMergerConfig::ESegmentsSort;

namespace {
    struct TInputSegment {
        TString DirName;
        ui32 DocCount;
        ui32 SortingKey;
        bool IsSearchable;
    public:
        TInputSegment() = default;

        TInputSegment(const TString& dirName, ui32 docCount)
            : DirName(dirName)
            , DocCount(docCount)
            , SortingKey()
            , IsSearchable(IsIndexSearchable(dirName))
        {
            Y_VERIFY(IsBaseName(dirName));
        }

    public:
        static bool IsIndexFinal(const TString& basename) {
            return NRTYServer::HasIndexNamePrefix(basename, DIRPREFIX_INDEX) || NRTYServer::HasIndexNamePrefix(basename, DIRPREFIX_PREP);
        }

        static bool IsIndexSearchable(const TString& basename) {
            return !NRTYServer::HasIndexNamePrefix(basename, DIRPREFIX_PREP);
        }

        static bool IsBaseName(const TString& dirName) {
             return TFsPath(dirName).Basename() == dirName;
        }
    };

    using TSortedSegments = TVector<TInputSegment>;

    //
    // Strategies of merging (which segments to merge first, what the order should be)
    //
    class IMergeStrategy {
    public:
        virtual ~IMergeStrategy() = default;

        virtual void Sort(TSortedSegments::iterator begin, TSortedSegments::iterator end) const = 0;
        virtual TSortedSegments::const_iterator SelectLessNeeded(const TSortedSegments& sortedSegments) const = 0;

        static THolder<IMergeStrategy> Construct(ESegmentsSort mode);
    };

    template <ESegmentsSort Mode>
    class TSort final : public IMergeStrategy {
        void Sort(TSortedSegments::iterator begin, TSortedSegments::iterator end) const override;
        TSortedSegments::const_iterator SelectLessNeeded(const TSortedSegments& sortedSegments) const override;
    };

    template<>
    void TSort<ESegmentsSort::Size>::Sort(TSortedSegments::iterator begin, TSortedSegments::iterator end) const {
        // Start with the smallest segments (default mode)
        SortBy(begin, end,
                [](const TInputSegment& s) {
                    return s.DocCount;
                });
    }

    template<>
    TSortedSegments::const_iterator TSort<ESegmentsSort::Size>::SelectLessNeeded(const TSortedSegments& sortedSegments) const {
        // exclude the biggest searchable segment from the list (default mode)
        auto i = std::find_if(sortedSegments.crbegin(), sortedSegments.crend(), [](const TInputSegment& v) {
            return v.IsSearchable;
        });
        if (i == sortedSegments.rend())
            return sortedSegments.end();
        return std::next(i).base();
    }

    template<>
    void TSort<ESegmentsSort::Id>::Sort(TSortedSegments::iterator begin, TSortedSegments::iterator end) const {
        // Start with the oldest segments, but always put the searchable segments first (required for Jupi wad merge)
        auto sortingKey = [](const TString& basename) {
            size_t shard;
            i32 segmentNo;
            if (!NRTYServer::GetShardAndSegmentId(basename, shard, segmentNo) || segmentNo < 0) {
                return Max<ui32>();
            };
            Y_UNUSED(shard);
            return (ui32)segmentNo;
        };

        auto sortingPred =[](const TInputSegment& a, const TInputSegment& b) {
            if (a.IsSearchable != b.IsSearchable)
                return a.IsSearchable;
            return a.SortingKey < b.SortingKey;
        };

        for (TSortedSegments::iterator p = begin; p != end; ++p) {
            p->SortingKey = sortingKey(p->DirName);
        }

        ::Sort(begin, end, sortingPred);
    }

    template<>
    TSortedSegments::const_iterator TSort<ESegmentsSort::Id>::SelectLessNeeded(const TSortedSegments& sortedSegments) const {
        // exclude the oldest searchable segment from the list
        auto i = std::find_if(sortedSegments.cbegin(), sortedSegments.cend(), [](const TInputSegment& v) {
            return v.IsSearchable;
        });
        return i;
    }

    THolder<IMergeStrategy> IMergeStrategy::Construct(ESegmentsSort mode) {
        switch(mode) {
            case ESegmentsSort::Size:
                return MakeHolder<TSort<ESegmentsSort::Size>>();
            case ESegmentsSort::Id:
                return MakeHolder<TSort<ESegmentsSort::Id>>();
        }
    }
}

//
// TAnalyzerAgent
//
bool TAnalyzerAgent::RemoveEmptyIndexes() {
    bool result = false;
    TGuard<TMutex> g(MutexIndexesProcessing);
    DEBUG_LOG << "Empty indexes clearing for " << Path << "..." << Endl;
    TVector<TString> indexes = Merger.GetIndexStorage().GetFinalIndexes(Path);
    for (auto&& i : indexes) {
        ui32 docNum = Merger.GetIndexStorage().GetDocsCount(i, false);
        if (!docNum) {
            size_t shard = NRTYServer::GetShard(i);
            if (ActiveTasks[shard])
                continue;
            INFO_LOG << "Index " << Path << "/" << i << " removing as empty ... locking ..." << Endl;
            auto constructionLock = TIndexConstructionLocks::Start(i);
            if (!!constructionLock) {
                INFO_LOG << "Index " << Path << "/" << i << " removing as empty ..." << Endl;
                Merger.GetIndexStorage().RemoveIndex(Path + "/" + i, true);
                INFO_LOG << "Index " << Path << "/" << i << " removed as empty ... OK" << Endl;
            }
            result = true;
        }
    }
    DEBUG_LOG << "Empty indexes clearing for " << Path << "...OK" << Endl;
    return result;
}


bool TAnalyzerAgent::BuildTasksForDirectory() {
    TGuard<TMutex> g(MutexIndexesProcessing);
    INFO_LOG << "TAnalyzerAgent::BuildTasksForDirectory ..." << Endl;

    TSortedSegments sortedSegments;
    THolder<IMergeStrategy> sorter = IMergeStrategy::Construct(SegmentsSort);

    TFileEntitiesList dirs(TFileEntitiesList::EM_DIRS);
    dirs.Fill(Path, "", "", 1);
    const char *dirName = nullptr;
    while ((dirName = dirs.Next()) != nullptr) {
        Y_ASSERT(TInputSegment::IsBaseName(dirName));
        if (!TInputSegment::IsIndexFinal(dirName))
            continue;

        const ui32 docCount = Merger.GetIndexStorage().GetDocsCount(dirName, false);
        if (docCount == Max<ui32>())
            continue;

        TInputSegment input(dirName, docCount);
        INFO_LOG << "Index " << input.DirName << " with " << input.DocCount << " documents checking for merging" << Endl;
        sortedSegments.emplace_back(std::move(input));
    }
    // With the default option values, this Sort() means that the selection process takes the smallest, not the oldest, segments first (reasons unclear, legacy)
    // see IMergeStrategy::Sort() for more
    sorter->Sort(sortedSegments.begin(), sortedSegments.end());

    TSortedSegments segmentsToMerge;
    for (NRTYServer::TShardsConfig::TLocalShards::const_iterator s = Config.GetShardsConfig().LocalShards.begin(); s != Config.GetShardsConfig().LocalShards.end(); ++s) {
        if (ActiveTasks[*s])
            continue;
        segmentsToMerge.clear();
        unsigned mergedDocsNum = 0;
        unsigned countSegments = 0;
        unsigned countSearchableToMerge = 0;
        unsigned countSearchableNotToMerge = 0;
        TStringStream ss;
        ss << "(";
        for (size_t i = 0; i < sortedSegments.size() && mergedDocsNum < GetMaxDocs(); ++i) {
            const TInputSegment& segment = sortedSegments[i];
            const TString& indexDirName = segment.DirName;
            if (NRTYServer::GetShard(indexDirName) == *s) {
                const unsigned docsNum = segment.DocCount;

                if (mergedDocsNum + docsNum <= GetMaxDocs() && docsNum) {
                    ss << docsNum << "/" << indexDirName << "+";
                    segmentsToMerge.push_back(segment);
                    mergedDocsNum += docsNum;
                    if (segment.IsSearchable) {
                        countSearchableToMerge++;
                    }
                } else if (docsNum) {
                    if (segment.IsSearchable) {
                        countSearchableNotToMerge++;
                    }
                }
                countSegments++;
            }
        }
        ss << ") ";

        // Apply Merger.MaxSegments
        //
        // Technically, Merger.MaxSegments is a combination of the two following conditions:
        // a) MaxSegments is "the minimal count of searchable segments that should exist after the merge" (hence the removal below), and
        // b) (MaxSegments+1) is "the minimal total count of segments, including empty (!), that can trigger creation of a merge task"
        //
        // Please note that Merger.MaxSegments is NOT a limit on the number of inputs in a merge task. When a user sets MaxSegments == 4, they expect
        // that 4 "balanced" segments of the same size will appear. This is not always the case (see SAAS-5303)

        ui32 countSearchableAfterMerge = 1 + countSearchableNotToMerge;
        for ( ; countSearchableAfterMerge < MaxSegments; countSearchableAfterMerge++) {
            // this excludes the last final index from the list ("last" means the biggest with the default value of SegmentsSort)
            TSortedSegments::const_iterator leastNeeded = sorter->SelectLessNeeded(segmentsToMerge);
            if (leastNeeded == segmentsToMerge.end())
                break;

            Y_VERIFY(leastNeeded->IsSearchable);
            Y_VERIFY(countSearchableToMerge > 0);

            segmentsToMerge.erase(leastNeeded);
            countSearchableToMerge--;
        }

        // Fill taskForMerge
        TVector<TString> taskForMerge;
        mergedDocsNum = 0;
        for (auto&& i : segmentsToMerge) {
            ss << i.DirName << "/" << i.DocCount << "+";
            taskForMerge.push_back(i.DirName);
            mergedDocsNum += i.DocCount;
        }

        if (taskForMerge.ysize() > 0 && mergedDocsNum > 0) {
            Y_ASSERT(countSegments > 0);
            bool needMerge = (taskForMerge.ysize() > 1 && countSegments > MaxSegments)
                || countSearchableToMerge == 0;
            if (needMerge) {
                INFO_LOG << "TAnalyzerAgent::BuildTasksForDirectory creating task: Segments count: " << taskForMerge.ysize() << ", DocsCount: " << mergedDocsNum << ", Segments: " << ss.Str() << Endl;
                SendGlobalMessage<TUniversalAsyncMessage>("mergerAnalyzerAddTask"); // testability hook
                const bool taskCreated = Merger.AddTask(*s, taskForMerge, Path, *this, LockPolicy, Realm == NRTYServer::ERealm::Realtime);
                if (!taskCreated)
                    WARNING_LOG << "TAnalyzerAgent::BuildTasksForDirectory failed to AddTask. The task is not created." << Endl;
            }
        } else if (countSegments > MaxSegments) {
            ERROR_LOG << "MaxSegments exceeded, but merge task not created: " << countSegments << ">" << MaxSegments << Endl;
            TSaasRTYServerSignals::OnCannotMerge(RealmConfigName, countSegments - MaxSegments);
        }
        ShardNewSegments[*s] = 0;
    }
    INFO_LOG << "TAnalyzerAgent::BuildTasksForDirectory ... OK" << Endl;
    return true;
}

ui32 TAnalyzerAgent::GetMaxDocs() {
    return std::min<unsigned>(Config.GetMergerConfig().MaxDocumentsToMerge, 1 << DOC_LEVEL_Bits);
}

class TKpsIterator : public TAtomicRefCount<TKpsIterator> {
public:
    typedef TIntrusivePtr<TKpsIterator> TPtr;
    TKpsIterator(TIndexControllerPtr index, bool isPrefixed)
        : Index(index)
        , Yndex(index->GetIndexData())
        , PruningCalcer(index->CreatePruningCalcer())
        , IsPrefixed(isPrefixed)
    {
        if (Yndex)
            Next();
    }

    double PruningRank() const {
        return PruningCalcer ? PruningCalcer->PruningRank(Docid()) : 0;
    }

    ui32 Docid() const {
        return PosIterator.Doc();
    }

    ui64 Kps() const {
        return CurrentKps;
    }

    bool Valid() const {
        return Yndex && PosIterator.Valid();
    }

    void NextKps() {
        if (PosIterator.Valid()) {
            PosIterator = TPosIterator<>();
        }
        Next();
    }

    void Next() {
        while (true) {
            if (PosIterator.Valid()) {
                PosIterator.Next();
            } else {
                ++Pos;
                const YxRecord* rec = Yndex->EntryByNumber(RequestContext, Pos, Block);
                if (!rec)
                    break;
                if (rec->TextPointer[0] < '(')
                    continue;
                if (rec->TextPointer[0] > '(')
                    break;
                TStringBuf zone;
                if (IsPrefixed){
                    CurrentKps = DecodePrefix(rec->TextPointer);
                    zone = TStringBuf(rec->TextPointer + 18, ZoneName.size());
                } else {
                    zone = TStringBuf(rec->TextPointer + 1, ZoneName.size());
                }
                if (!IsPrefixed && zone > ZoneName)
                    break;
                if (zone != ZoneName)
                    continue;
                PosIterator.Init(*Yndex, rec->Offset, rec->Length, rec->Counter, RH_DEFAULT);
            }
            if (PosIterator.Valid() && !Index->IsRemoved(PosIterator.Doc()))
                break;
        }
    }

    void Remove() {
        Index->RemoveDocIds(TVector<ui32>(1, PosIterator.Doc()));
    }

private:
    TIndexControllerPtr Index;
    const IKeysAndPositions* Yndex;
    THolder<TPruningConfig::ICalcer> PruningCalcer;
    TRequestContext RequestContext;
    i32 Pos = -1;
    i32 Block = UNKNOWN_BLOCK;
    TPosIterator<> PosIterator;
    ui64 CurrentKps = 0;
    bool IsPrefixed;
    static const TStringBuf ZoneName;
};

const TStringBuf TKpsIterator::ZoneName("main_content");

struct TDocWorse {
    bool operator()(TKpsIterator::TPtr l, TKpsIterator::TPtr r) {
        return l->PruningRank() < r->PruningRank();
    }
};

void TAnalyzerAgent::CheckPerKpsDocCountRestriction() {
    if (!Config.GetMergerConfig().MaxDocsPerKps.Enabled())
        return;

    TGuard<TMutex> g(MutexIndexesProcessing);
    INFO_LOG << "TAnalyzerAgent::CheckPerKpsDocCountRestriction ..." << Endl;
    TVector<TString> indexes = Merger.GetIndexStorage().GetFinalIndexes(Path);
    typedef TVector<TKpsIterator::TPtr> TKpsVector;
    TKpsVector kpsIterators;
    for (const auto& s : Config.GetShardsConfig().LocalShards) {
        if (ActiveTasks[s])
            continue;
        for (const TString& index : indexes) {
            if (NRTYServer::GetShard(index) == s) {
                TIndexControllerPtr controller = Merger.GetIndexStorage().GetIndexController(index);
                TKpsIterator::TPtr iter = MakeIntrusive<TKpsIterator>(controller, Config.IsPrefixedIndex);
                if (iter->Valid())
                    kpsIterators.push_back(iter);
            }
        }
    }

    while (!kpsIterators.empty()) {
        StableSort(kpsIterators.begin(), kpsIterators.end(),
            [](TKpsIterator::TPtr l, TKpsIterator::TPtr r) {
                return l->Kps() < r->Kps();
            }
        );

        ui64 kps = kpsIterators.front()->Kps();
        if (ui64 allowedCount = Config.GetMergerConfig().MaxDocsPerKps.Get(kps)) {
            auto kpsEnd = UpperBound(kpsIterators.begin(), kpsIterators.end(), kps,
                    [](ui64 kps, TKpsIterator::TPtr iter) {
                    return kps < iter->Kps();
                }
            );
            TKpsVector oneKps(kpsIterators.begin(), kpsEnd);
            TDocWorse comparier;
            MakeHeap(oneKps.begin(), oneKps.end(), comparier);
            while (!oneKps.empty() && allowedCount) {
                --allowedCount;
                PopHeap(oneKps.begin(), oneKps.end(), comparier);
                oneKps.back()->Next();
                if (oneKps.back()->Valid() && oneKps.back()->Kps() == kps)
                    PushHeap(oneKps.begin(), oneKps.end(), comparier);
                else
                    oneKps.pop_back();
            }
            if (!allowedCount) {
                ui64 removed = 0;
                for (auto i : oneKps) {
                    for (;i->Valid() && oneKps.back()->Kps() == kps;i->Next()) {
                        i->Remove();
                        ++removed;
                    }
                }
                INFO_LOG << "TAnalyzerAgent::CheckPerKpsDocCountRestriction ..." << removed << " docs removed from kps " << kps << Endl;
            }
        } else {
            for (auto i : kpsIterators) {
                if(i->Kps() == kps) {
                    i->NextKps();
                } else {
                    break;
                }
            }
        }
        TKpsVector newIterators;
        newIterators.reserve(kpsIterators.size());
        for (auto i : kpsIterators)
            if (i->Valid())
                newIterators.push_back(i);
        kpsIterators = newIterators;
    }
    INFO_LOG << "TAnalyzerAgent::CheckPerKpsDocCountRestriction ...OK" << Endl;
}

bool TAnalyzerAgent::Process(IMessage* message) {

    TMessageRemoveEmptyIndexes* messRemoveEmptyIndexes = dynamic_cast<TMessageRemoveEmptyIndexes*>(message);
    if (messRemoveEmptyIndexes) {
        RemoveEmptyIndexes();
        return true;
    }

    TMessageRegisterIndex* messRegisterIndex = dynamic_cast<TMessageRegisterIndex*>(message);
    if (messRegisterIndex) {
        if (messRegisterIndex->GetIndexController()->GetType() & IIndexController::FINAL_MASK) {
            auto newIndexPath = messRegisterIndex->GetIndexController()->Directory().PathName();
            if (TFsPath(Path).IsContainerOf(TFsPath(newIndexPath))) {
                TGuard<TMutex> g(MutexCounter);
                const NRTYServer::TShard shard = NRTYServer::GetShard(newIndexPath);
                INFO_LOG << "Merger task for shard " << shard << " status = " << ActiveTasks[shard] << ": new segments: " << ShardNewSegments[shard] << " + 1" << Endl;
                ShardNewSegments[shard]++;
                if (!ActiveTasks[shard]) {
                    if (MergerPolicy->IsDependOnExternalEvents()) {
                        MergerPolicy->OnNewIndex();
                        CondVarWaiting.Signal();
                    }
                }
            }
        }
        return true;
    }

    TMessageCreateMergerTasks* messCreateMergerTasks = dynamic_cast<TMessageCreateMergerTasks*>(message);
    if (messCreateMergerTasks) {
        if (Path == Config.GetRealmListConfig().GetMainRealmConfig().RealmDirectory) {
            CheckPerKpsDocCountRestriction();
            RemoveEmptyIndexes();
            TGuard<TMutex> g(MutexCounter);
            BuildTasksForDirectory();
            return true;
        }
    }
    return false;
}

void TAnalyzerAgent::OnWatchdogOption(const TString&, const TString& value) {
    // only one Key is subscribed to
    Disabled = IsTrue(value);
}

void TAnalyzerAgent::Process(void* /*threadSpecificResource*/) {
    ThreadDisableBalloc();
    while (!DoStopSignal) {
        INFO_LOG << "Tasks construction logic start" << Endl;
        CheckPerKpsDocCountRestriction();
        RemoveEmptyIndexes();
        {
            TGuard<TMutex> g(MutexCounter);
            if (!Disabled)
                BuildTasksForDirectory();

            INFO_LOG << "Merging analyzer sleeping..." << Endl;
            if (!DoStopSignal) {
                bool waitOk = CondVarWaiting.WaitD(MutexCounter, MergerPolicy->GetWaitingDeadline());
                if (waitOk && MergerPolicy->IsDependOnExternalEvents()) {
                    TInstant sleepingDeadline = MergerPolicy->GetSleepingDeadline(); // TInstant::Zero() allowed
                    while (!DoStopSignal)
                        if (!CondVarWaiting.WaitD(MutexCounter, sleepingDeadline))
                            break; // sleepingDeadline reached
                }
            }
        }
        INFO_LOG << "Merging analyzer wake up" << Endl;
    }
}

TAnalyzerAgent::TAnalyzerAgent(const TString& path, ui32 maxSegments, IMerger& merger,
    const TRTYServerConfig& config, IMergerPolicyBehavior::TPtr mergerPolicy,
    IMergerLockPolicy::TPtr lockPolicy, NRTYServer::ERealm realm, const TString& realmConfigName)
    : Path(path)
    , MaxSegments(maxSegments)
    , Disabled(false)
    , Merger(merger)
    , MergerPolicy(mergerPolicy)
    , LockPolicy(lockPolicy)
    , Realm(realm)
    , RealmConfigName(realmConfigName)
    , SegmentsSort(config.GetMergerConfig().SegmentsSort)
    , Config(config)
{
    CHECK_WITH_LOG(!!mergerPolicy);
    ShardNewSegments.resize(Config.GetShardsConfig().Number, 0);
    ActiveTasks.resize(Config.GetShardsConfig().Number, false);
    RegisterGlobalMessageProcessor(this);
}

TAnalyzerAgent::~TAnalyzerAgent() {
    UnregisterGlobalMessageProcessor(this);
}

void TAnalyzerAgent::SubscribeToWatchdog(IWatchdogOptions& w) {
    //
    // The feature may be used through http handle:
    // * Do not start new merger tasks:
    //     ?command=set_its_override&key=rty.merger.disabled&value=1
    // * Restore normal mode:
    //     ?command=set_its_override&key=rty.merger.disabled&revert=1
    //
    Watchdog = w.Subscribe(this, "rty.merger.disabled"); // w.Subscribe will cause OnWatchdogOption() calls with the initial values.
}

void TAnalyzerAgent::WaitMergingFinished() const {
    TGuard<TMutex> g(MutexCounter);
    while (TasksCount && !DoStopSignal) {
        CondVarCounter.WaitI(MutexCounter);
    }
}

void TAnalyzerAgent::Stop() {
    DoStopSignal = true;
    CondVarWaiting.Signal();
    CondVarCounter.Signal();
    Watchdog.Drop();
}

void TAnalyzerAgent::ModifyStateOnMergerTaskFinish(ui32 shard) {
    TGuard<TMutex> g(MutexCounter);
    CHECK_WITH_LOG(ActiveTasks[shard]);
    CHECK_WITH_LOG(ActiveTasks.size() > shard);
    INFO_LOG << "Merger task for shard " << shard << " finished: new segments: " << ShardNewSegments[shard] << Endl;
    ActiveTasks[shard] = false;
    MergerPolicy->OnMergerTaskFinish(ShardNewSegments[shard], RealmConfigName);
    if (ShardNewSegments[shard]) {
        if (MergerPolicy->IsDependOnExternalEvents())
            CondVarWaiting.Signal();
    }
    --TasksCount;
    TMessageOnMergerTaskFinish message;
    SendGlobalMessage(message);
    CondVarCounter.Signal();
}

void TAnalyzerAgent::OnMergerTaskFinish(const IMergerTask* task) {
    const ui32 shard = task->GetShardNumber();
    ModifyStateOnMergerTaskFinish(shard);
    if (const auto duration = task->GetMainStageDuration(); task->IsFinished() && duration) {
        TSaasRTYServerSignals::OnMergerTaskFinish(RealmConfigName, task->GetMainStageDuration());
    }
    TSaasRTYServerSignals::OnCannotMerge(RealmConfigName, 0);
}

void TAnalyzerAgent::OnMergerTaskFailed() {
    TSaasRTYServerSignals::OnMergerTaskFailed(RealmConfigName);
}

void TAnalyzerAgent::OnMergerTaskStart(const IMergerTask* task) {
    TSaasRTYServerSignals::OnMergerTaskStart(RealmConfigName);
    const ui32 shard = task->GetShardNumber();
    TGuard<TMutex> g(MutexCounter);
    CHECK_WITH_LOG(!ActiveTasks[shard]);
    CHECK_WITH_LOG(ActiveTasks.size() > shard);
    INFO_LOG << "Merger task for shard " << shard << " started: new segments: " << ShardNewSegments[shard] << Endl;
    ActiveTasks[shard] = true;
    TMessageOnMergerTaskStart message;
    SendGlobalMessage(message);
    ++TasksCount;
}
