#include "detach_processor.h"
#include "synchronizer.h"
#include "config.h"
#include <saas/library/yt/file/yt_file_dumper.h>
#include <saas/protos/yt.pb.h>
#include <saas/library/yt/common/yt_blob.h>
#include <saas/rtyserver/common/sharding.h>
#include <saas/rtyserver/config/realm_config.h>
#include <saas/util/external/sky.h>
#include <saas/util/system/copy_recursive.h>
#include <saas/rtyserver/indexer_core/index_dir.h>
#include <saas/rtyserver/indexer_server/indexer_server.h>
#include <saas/rtyserver/final_index/final_index_normalizer.h>
#include <util/system/fstat.h>
#include <util/folder/filelist.h>
#include <library/cpp/balloc/optional/operators.h>
#include <mapreduce/yt/interface/client.h>

namespace NRTYServer {


    void TDetachProcessor::Stop() {
        RigidStop = true;
    }

    void TDetachProcessor::ProcessTask(const TDetachTask::TContext& context, TAsyncTaskExecutor::TTask& task) {
        task.GetReply()->InsertValue("stage", "IN_QUEUE");
        TGuard<TMutex> g(Mutex);
        TMap<TString, TVector<TString>> segments;
        ThreadDisableBalloc();
        task.GetReply()->InsertValue("stage", "INDEXATION_STOP");
        TIndexationStopGuard isg;
        task.GetReply()->InsertValue("stage", "MERGER_STOP_WAITING");
        TMergerStopGuard msg(false);
        if (context.ClearPrevious && TFsPath(context.DetachDirectory).Exists()) {
            TDirsList dirs;
            dirs.Fill(context.DetachDirectory);
            while (const char* dir = dirs.Next()) {
                TString fullPath = context.DetachDirectory + "/" + (TString)dir;
                NOTICE_LOG << "Delete previous detach directory " << (TString)fullPath << Endl;
                TFsPath(fullPath).ForceDelete();
            }
        }
        TDetachTask* origDetachTask = new TDetachTask(context);
        IMergerTask::TPtr detachTask = origDetachTask;
        task.GetReply()->InsertValue("stage", "DECODER_BUILD");
        detachTask->BuildDecoder(TMergerEngine::GetMerger()->GetIndexStorage());
        task.GetReply()->InsertValue("stage", "DETACHING");
        TFsPath destTo(context.DetachDirectory + "/" + detachTask->GetName());
        destTo.MkDirs();
        {
            TUnbufferedFileOutput fo(destTo.GetPath() + "/description");
            fo << origDetachTask->GetDescription();
        }
        if (origDetachTask->NeedToMerge())
            TMergerEngine::GetMerger()->DoTask(detachTask, context.StopFlag);

        IIndexStorage& storage = TMergerEngine::GetMerger()->GetIndexStorage();

        TDetachTask::TSegmentsForCopy copiedSegments;
        for (TDetachTask::TSegmentsForCopy::const_iterator i = origDetachTask->GetSegmentsForCopy().begin(); i != origDetachTask->GetSegmentsForCopy().end(); ++i) {
            TFsPath path(context.Config.GetRealmListConfig().GetMainRealmConfig().RealmDirectory);
            ui32 segmentIndex = (i->first + 1) * 10000;
            for (auto segment : i->second) {
                const bool isPrep = NRTYServer::HasIndexNamePrefix(segment, DIRPREFIX_PREP);
                const TStringBuf& indexTypePrefix = isPrep ? DIRPREFIX_PREP : DIRPREFIX_INDEX;
                const TString newName = TClusterInfo(NRTYServer::GetShard(segment), segmentIndex++).FormatName(indexTypePrefix);
                const TFsPath newPath = context.DetachDirectory + "/" + detachTask->GetName() + "/" + newName;
                const TFsPath currentPath = path / segment;

                NUtil::CopyRecursive(currentPath, newPath);
                (newPath / "source_indexes").ForceDelete();
                TIndexControllerPtr index = storage.GetIndexController(currentPath);
                if (!!index) {
                    if (!TFinalIndexNormalizer::BuildServiceFiles(TPathName(newPath), context.Config)) {
                        ERROR_LOG << "Normalization over copied index: " << newPath << " has failed" << Endl;
                        if (!isPrep) {
                            ythrow yexception() << "Normalization over copied index has failed";
                        }
                    }
                }
                copiedSegments[i->first].push_back(newName);
                INFO_LOG << "Copy " << currentPath << " to " << context.DetachDirectory + "/" + detachTask->GetName() + "/" + newName << Endl;
            }
        }
        if (context.Share) {
            task.GetReply()->InsertValue("stage", "SHARING");
            for (ui32 i = 0; i < context.ShardIntervals.size(); ++i) {
                const TVector<TClusterInfo>& clusters = origDetachTask->GetClusters(i);
                TString torrentResult = "EMPTY_";
                TVector<TString> clusterSegments;
                for (ui32 j = 0; j < clusters.size(); ++j)
                    clusterSegments.push_back(detachTask->GetDestSegments()[clusters[j].Id]);
                TDetachTask::TSegmentsForCopy::const_iterator dests = copiedSegments.find(i);
                if (dests != copiedSegments.end())
                    clusterSegments.insert(clusterSegments.end(), dests->second.begin(), dests->second.end());
                if (!clusterSegments.empty()) {
                    segments[context.ShardIntervals[i].ToString()] = clusterSegments;
                    INFO_LOG << "Sharing " << destTo.GetPath() << ": " << JoinStrings(clusterSegments, ",") << Endl;
                    NUtil::TSkynet::Share(destTo.GetPath(), clusterSegments, torrentResult);
                }
                if (context.ShardIntervals.size() == 1)
                    task.GetReply()->InsertValue("id_res", torrentResult.substr(0, torrentResult.size() - 1));
                task.GetReply()->InsertValue("id_res_" + context.ShardIntervals[i].ToString(), torrentResult.substr(0, torrentResult.size() - 1));
                DEBUG_LOG << context.ShardIntervals[i].ToString() << " -> " << torrentResult << Endl;
            }
        }
        if (context.UploadToYt) {
            task.GetReply()->InsertValue("stage", "UPLOADING");
            auto ytClient = NYT::CreateClient(context.YtProxy);
            INFO_LOG << "Detaching to yt begins" << Endl;
            auto tx = ytClient->StartTransaction();
            tx->Create(context.YtPath, NYT::NT_TABLE, NYT::TCreateOptions().IgnoreExisting(true).Recursive(true));

            auto path = NYT::TRichYPath(context.YtPath).Append(false);
            auto writer = NYT::TTableWriterPtr<NYT::Message>(tx->CreateTableWriter<NSaas::TYTBlobBase>(path));
            TVector<TString> segmentsToUpload(detachTask->GetDestSegments());
            for (const auto& segmentNames : copiedSegments) {
                segmentsToUpload.insert(segmentsToUpload.end(), segmentNames.second.begin(), segmentNames.second.end());
            }

            int segmentId = 0;
            auto shardId = context.ShardIntervals[0];
            for (const auto& segment : segmentsToUpload) {
                TYTChunkedOutput::TConstructionContext dumperContext({writer.Get(), shardId, 0, segmentId++});
                TYTFileDumper dumper(segment, dumperContext);
                INFO_LOG << "Dumping directory " << segment << Endl;
                dumper.DumpDirectory(destTo / segment);
            }
            writer->Finish();
            tx->Commit();
        }


        task.GetReply()->InsertValue("stage", "DONE");
        task.GetReply()->InsertValue("task_name", detachTask->GetName());
    }

    TDetachProcessor::TDetachProcessor(TSynchronizer* synchronizer)
        : ISynchronizerProcessor(synchronizer)
    {
        RigidStop = false;
        NOTICE_LOG << "Remove detach cache" << Endl;
        size_t checkTs = Now().Seconds();
        try {
            TDirsList dirs;
            dirs.Fill(synchronizer->GetConfig()->DetachPath);
            const char *dir;
            while (dir = dirs.Next()) {
                TString fullPath = synchronizer->GetConfig()->DetachPath + "/" + (TString)dir;
                TFileStat fs(fullPath);
                if (checkTs - fs.MTime > 24 * 60 * 60) {
                    NOTICE_LOG << "Detach directory " << (TString)fullPath << " timeout expired (" << checkTs << "/" << fs.MTime << "/" << checkTs - fs.MTime << ")" << Endl;
                    TFsPath(fullPath).ForceDelete();
                } else {
                    NOTICE_LOG << "Detach directory " << (TString)dir << " is actual yet (" << checkTs << "/" << fs.MTime << "/" << checkTs - fs.MTime << ")" << Endl;
                }
            }
            TFsPath(synchronizer->GetConfig()->DetachPath).MkDirs();
            NOTICE_LOG << "Remove detach cache OK" << Endl;
        } catch (...) {
            ERROR_LOG << "Remove detach cache FAILED" << CurrentExceptionMessage() << Endl;
        }
    }

    bool TDetachProcessor::ProcessRequest(const TCgiParameters& cgi, TStringBuf postBuffer, TAsyncTaskExecutor::TTask& task) {
        Y_UNUSED(postBuffer);
        DEBUG_LOG << "Start detach request processing " << cgi.Print() << Endl;

        TVector<TString> minShardS = SplitString(cgi.Get("min_shard"), ",");
        TVector<TString> maxShardS = SplitString(cgi.Get("max_shard"), ",");

        TDetachTask::TContext dtCxt(Synchronizer->GetRTYConfig());
        dtCxt.Sharding = NSaas::TShardsDispatcher::TContext::FromString(cgi.Get("sharding_type"));

        if (minShardS.size() != maxShardS.size()) {
            ythrow yexception() << "incorrect &min_shard and &max_shard sizes";
        }
        for (ui32 i = 0; i < minShardS.size(); ++i) {
            DEBUG_LOG << "Check interval " << minShardS[i] << "-" << maxShardS[i] << Endl;
            if (!minShardS[i] && !maxShardS[i])
                continue;
            try {
                ::TDetachTask::TShardInterval si(
                        FromString<NSearchMapParser::TShardIndex>(minShardS[i]),
                        FromString<NSearchMapParser::TShardIndex>(maxShardS[i])
                    );
                for (ui32 j = 0; j < dtCxt.ShardIntervals.size(); ++j)
                    if (si.Intersection(dtCxt.ShardIntervals[j]))
                        ythrow yexception() << "intervals incorrect (intersection is not empty)";
                dtCxt.ShardIntervals.push_back(si);
            }
            catch (...) {
                DEBUG_LOG << "Check interval " << minShardS[i] << "-" << maxShardS[i] << "...FAILED" << Endl;
                ythrow yexception() << "incorrect &min_shard or &max_shard " << cgi.Get("min_shard") << "/" << cgi.Get("max_shard") << " : " << CurrentExceptionMessage();
            }
            DEBUG_LOG << "Check interval " << minShardS[i] << "-" << maxShardS[i] << "...OK" << Endl;
        }
        dtCxt.DetachDirectory = Synchronizer->GetConfig()->DetachPath;
        dtCxt.ClearPrevious = Synchronizer->GetConfig()->DetachClearPreviousResults;
        dtCxt.DestSegmentSize = Synchronizer->GetConfig()->DetachedSegmentSize;
        dtCxt.DestSegmentSizeDeviation = Synchronizer->GetConfig()->DetachedSegmentSizeDeviation;
        dtCxt.StopFlag = &RigidStop;
        dtCxt.Share = !cgi.Has("no_share") || !FromString<bool>(cgi.Get("no_share"));
        dtCxt.UploadToYt = cgi.Has("upload_to_yt") && FromString<bool>(cgi.Get("upload_to_yt"));
        if (cgi.Has("yt_proxy")) {
            dtCxt.YtProxy = cgi.Get("yt_proxy");
        }
        if (cgi.Has("yt_path")) {
            dtCxt.YtPath = cgi.Get("yt_path");
        }
        ProcessTask(dtCxt, task);
        DEBUG_LOG << "Finish detach request processing " << cgi.Print() << Endl;
        return true;
    }

}
