#include "rty_storage.h"

#include <saas/util/logging/trace.h>
#include <search/idl/meta.pb.h>

#include <library/cpp/deprecated/atomic/atomic.h>
#include <library/cpp/http/io/stream.h>
#include <google/protobuf/io/coded_stream.h>

#include <util/charset/utf8.h>
#include <util/digest/fnv.h>
#include <util/generic/hash.h>
#include <util/string/printf.h>
#include <util/folder/path.h>
#include <util/folder/pathsplit.h>
#include <util/string/type.h>
#include <util/system/hostname.h>
#include <library/cpp/string_utils/quote/quote.h>
#include <library/cpp/string_utils/base64/base64.h>
#include <util/system/getpid.h>

namespace NSaas {

    IVersionedStorage::TFactory::TRegistrator<TRTYStorage> TRTYStorage::Registrator(IVersionedStorage::TOptions::TStorageType::RTY);

    ui32 TRTYStorage::ReadReply(const TString& query, TString& result) const {
        try {
            TNetworkAddress addr(Options.RTYStorage.SearchHost, Options.RTYStorage.SearchPort);
            TSocket s(addr);
            s.SetSocketTimeout(10, 0);
            s.SetZeroLinger();
            TString request = "/?" + query + "&ms=proto" + "&service=" + Options.RTYStorage.SearchService + "&kps=" + ToString(Options.RTYStorage.Prefix);
            DEBUG_LOG << "REQUEST: " << Options.RTYStorage.SearchHost << ":" << Options.RTYStorage.SearchPort << "/" << request << Endl;
            SendMinimalHttpRequest(s, GetHostName(), request);

            TSocketInput si(s);
            THttpInput input(&si);
            unsigned httpCode = ParseHttpRetCode(input.FirstLine());
            result = input.ReadAll();
            return httpCode;
        } catch (...) {
            ERROR_LOG << "Error on search: " << CurrentExceptionMessage() << Endl;
            return 0;
        }
    }

    bool TRTYStorage::SendRequest(const TString& request, NMetaProtocol::TReport& report) const {

        TString reply;

        ui32 code = ReadReply(request, reply);

        if (code != 200) {
            ERROR_LOG << "Errors while process query " << request << ", code=" << code << Endl;
            return false;
        } else {
            ::google::protobuf::io::CodedInputStream decoder((ui8*)reply.data(), reply.size());
            decoder.SetTotalBytesLimit(1 << 29);
            if (!report.ParseFromCodedStream(&decoder) || !decoder.ConsumedEntireMessage()) {
                ythrow yexception() << "Protobuf ParseFromArcadiaStream error";
            }
        }
        return true;
    }

    bool TRTYStorage::GetValue(const TString& key, TString& result, i64 version, bool lock) const {
        TAbstractLock::TPtr lockPtr = nullptr;
        if (lock) {
            TDebugLogTimer lockTimer("lock " + key);
            lockPtr = ZooStorage->ReadLockNode("lock" + TFsPath(key).Fix().GetPath());
        }
        try {
            NMetaProtocol::TDocument doc;
            if (!GetKeyData(key, doc, version, true)) {
                DEBUG_LOG << "can't get value by key " << key << Endl;
                return false;
            }
            for (unsigned a = 0; a < doc.GetArchiveInfo().GtaRelatedAttributeSize(); ++a) {
                const NMetaProtocol::TPairBytesBytes& attr = doc.GetArchiveInfo().GetGtaRelatedAttribute(a);
                if (attr.GetKey() == "p_data_base64") {
                    result = Base64Decode(attr.GetValue());
                    return true;
                }
                if (attr.GetKey() == "p_data") {
                    result = attr.GetValue();
                    if (result == "$$$empty$$$")
                        result = "";
                    return true;
                }
            }
        } catch (...) {
            ERROR_LOG << "Error on TRTYStorage::GetValue: " << CurrentExceptionMessage() << Endl;
        }
        DEBUG_LOG << "can't get value by key " << key << Endl;
        return false;
    }

    bool TRTYStorage::ExistsNode(const TString& key) const {
        try {
            NMetaProtocol::TDocument doc;
            if (!GetKeyData(key, doc, -1, false, false))
                return false;
            for (unsigned a = 0; a < doc.FirstStageAttributeSize(); ++a) {
                const NMetaProtocol::TPairBytesBytes& attr = doc.GetFirstStageAttribute(a);
                if (attr.GetKey() == "g_deleted") {
                    return !IsTrue(attr.GetValue());
                }
            }
        } catch (...) {
            ERROR_LOG << "Error on TRTYStorage::ExistsNode: " << CurrentExceptionMessage() << Endl;
        }
        return false;
    }

    bool TRTYStorage::RemoveNodeImpl(const TString& pkey) const {
        try {
            DEBUG_LOG << "REMOVE NODE: " << pkey << Endl;
            ui64 version = Now().MilliSeconds();
            TString key = "/" + pkey;
            TString fixedKey = TFsPath(key).Fix().GetPath();
            TString parent = TFsPath(key).Fix().Parent().GetPath();
            NSaas::TAction action;
            action.SetPrefix(Options.RTYStorage.Prefix).GetDocument().SetUrl(fixedKey + "$$" + Sprintf("ver%010lu", version))
                .AddGroupAttribute("g_version", version)
                .AddGroupAttribute("g_deleted", 1)
                .AddGroupAttribute("g_path", FnvHash<ui64>(fixedKey.data(), fixedKey.size()) / 2)
                .AddProperty("p_deleted", "1")
                .AddProperty("p_path", fixedKey)
                .AddSearchAttribute("s_deleted", "1")
                .AddSearchAttribute("s_type", "CONTENT")
                .AddSearchAttribute("s_parent", parent)
                .AddSearchAttribute("s_path", fixedKey);
            NSaas::TSendResult result = SendAction(action);
            if (result.GetCode() != NSaas::TSendResult::srOK) {
                ERROR_LOG << "Error on RemoveNode: " << result.GetMessage() << Endl;
                return false;
            }
            NSaas::TAction actionStructure;
            actionStructure.SetPrefix(Options.RTYStorage.Prefix).GetDocument().SetUrl(fixedKey)
                .AddGroupAttribute("g_version", version)
                .AddGroupAttribute("g_deleted", 1)
                .AddGroupAttribute("g_path", FnvHash<ui64>(fixedKey.data(), fixedKey.size()) / 2)
                .AddProperty("p_deleted", "1")
                .AddProperty("p_path", fixedKey)
                .AddSearchAttribute("s_deleted", "1")
                .AddSearchAttribute("s_type", "FILE")
                .AddSearchAttribute("s_type", "DIR")
                .AddSearchAttribute("s_parent", parent)
                .AddSearchAttribute("s_path", fixedKey);
            result = SendAction(actionStructure);
            if (result.GetCode() != NSaas::TSendResult::srOK) {
                ERROR_LOG << "Error on RemoveNode: " << result.GetMessage() << Endl;
                return false;
            }
            return true;
        } catch (...) {
            ERROR_LOG << "Error on RemoveNode: " << CurrentExceptionMessage() << Endl;
        }
        return false;
    }

    bool TRTYStorage::RemoveNode(const TString& pkey, bool /*withHistory*/) const {
        try {
            TVector<TString> nodes;
            if (GetNodes(pkey, nodes, true)) {
                for (auto&& i : nodes) {
                    if (!RemoveNode(pkey + "/" + i))
                        return false;
                }
            }
            if (!RemoveNodeImpl(pkey))
                return false;
            return true;
        } catch (...) {
            ERROR_LOG << "Error on RemoveNode: " << CurrentExceptionMessage() << Endl;
        }
        return false;
    }

    NSaas::TSendResult TRTYStorage::SendAction(const NSaas::TAction& action) const {
        NSaas::TSendResult result;
        ui32 att = 3;
        while (--att) {
            result = IndexingClient.SendJsonRef(action);
            if (result.GetCode() == NSaas::TSendResult::srOK || result.GetCode() == NSaas::TSendResult::srIncorrectMessage)
                break;
        }
        return result;
    }

    namespace {
        TAtomic versionsCounter = 0;
    }

    bool TRTYStorage::SetValue(const TString& pkey, const TString& value, bool storeHistory, bool lock, i64* versionOut) const {
        TAbstractLock::TPtr lockPtr = nullptr;
        TString key = "/" + pkey;
        DEBUG_LOG << "SET_VALUE: " << pkey << Endl;
        if (lock) {
            TDebugLogTimer lockTimer("lock " + key);
            lockPtr = ZooStorage->WriteLockNode("lock" + TFsPath(key).Fix().GetPath());
        }
        try {
            ui64 version;
            while (true) {
                version = Now().MilliSeconds();
                VERIFY_WITH_LOG(version < (Max<ui64>() >> 16), "Version too big, it may become cause of problems");
                TString threadHash = HostName() + ":" + ToString(GetPID()) + ":" + ToString(AtomicIncrement(versionsCounter));
                version = (version << 16) | (ComputeHash(threadHash) & (1 << 16 - 1));
                TString parent = TFsPath(key).Fix().Parent().GetPath();
                TString fixedKey = TFsPath(key).Fix().GetPath();
                NSaas::TAction action;
                action.SetPrefix(Options.RTYStorage.Prefix).GetDocument().SetUrl(fixedKey + (storeHistory ? ("$$" + Sprintf("ver%010lu", version)) : "_content"))
                    .SetVersion(version / Max<ui32>())
                    .SetTimestamp(version % Max<ui32>())
                    .AddGroupAttribute("g_version", version)
                    .AddGroupAttribute("g_deleted", 0)
                    .AddGroupAttribute("g_path", FnvHash<ui64>(fixedKey.data(), fixedKey.size()) / 2)
                    .AddProperty("p_deleted", "0")
                    .AddProperty("p_path", fixedKey)
                    .AddProperty("p_parent", parent)
                    .AddSearchAttribute("s_deleted", "0")
                    .AddSearchAttribute("s_type", "CONTENT")
                    .AddSearchAttribute("s_parent", parent)
                    .AddSearchAttribute("s_path", fixedKey);
                if (UTF8Detect(value.data(), value.size()) != NotUTF8) {
                    action.GetDocument().AddProperty("p_data", (!value) ? "$$$empty$$$" : value);
                } else {
                    TString b64Value = Base64Encode(value);
                    action.GetDocument().AddProperty("p_data_base64", b64Value);
                }
                NSaas::TSendResult result = SendAction(action);
                if (result.GetCode() == NSaas::TSendResult::srOK)
                    break;
                if (result.GetCode() == NSaas::TSendResult::srDeprecated)
                    continue;
                ERROR_LOG << "Error on SetValue: " << result.GetMessage() << Endl;
                return false;
            }
            if (versionOut)
                *versionOut = version;
            auto split = TFsPath(key).PathSplit();
            TString currentPath;
            TString predPath;
            for (ui32 i = 0; i < split.size(); ++i) {
                predPath = TFsPath(currentPath + "/").Fix().GetPath();
                currentPath = TFsPath(currentPath + "/" + split[i]).Fix().GetPath();
                NSaas::TAction actionStructure;
                actionStructure.SetPrefix(Options.RTYStorage.Prefix).GetDocument().SetUrl(currentPath)
                    .SetVersion(version / Max<ui32>())
                    .SetTimestamp(version % Max<ui32>())
                    .AddGroupAttribute("g_version", version)
                    .AddGroupAttribute("g_deleted", 0)
                    .AddGroupAttribute("g_path", FnvHash<ui64>(currentPath.data(), currentPath.size()) / 2)
                    .AddProperty("p_deleted", "0")
                    .AddProperty("p_path", currentPath)
                    .AddSearchAttribute("s_deleted", "0")
                    .AddSearchAttribute("s_type", (i == split.size() - 1) ? "FILE" : "DIR")
                    .AddSearchAttribute("s_parent", predPath)
                    .AddSearchAttribute("s_path", currentPath);
                NSaas::TSendResult result = SendAction(actionStructure);
                if (result.GetCode() != NSaas::TSendResult::srOK) {
                    ERROR_LOG << "Error on SetValue: " << result.GetMessage() << Endl;
                    return false;
                }
            }
            return true;
        } catch (...) {
            ERROR_LOG << "Error on SetValue: " << CurrentExceptionMessage() << Endl;
        }
        return false;
    }

    bool TRTYStorage::GetNodes(const TString& pkey, TVector<TString>& result, bool withDirs) const {
        result.clear();
        try {
            TString key = "/" + pkey;
            TSet<TString> objects;
            NMetaProtocol::TReport report;
            TString request = "s_parent:\"" + TFsPath(key).Fix().GetPath() + "\" (s_type:FILE" + (withDirs ? " | s_type:DIR)" : ")");
            UrlEscape(request);
            if (SendRequest("text=" + request + "&how=docid&numdoc=10000", report)) {
                if (report.GetGrouping().size() > 0) {
                    const NMetaProtocol::TGrouping& grouping(report.GetGrouping(0));
                    for (int i = 0; i < grouping.GetGroup().size(); i++) {
                        const NMetaProtocol::TGroup& group(grouping.GetGroup(i));
                        for (int d = 0; d < group.GetDocument().size(); d++) {
                            TString path;
                            bool notDeleted = true;
                            for (unsigned a = 0; a < group.GetDocument(d).GetArchiveInfo().GtaRelatedAttributeSize(); ++a) {
                                const NMetaProtocol::TPairBytesBytes& attr = group.GetDocument(d).GetArchiveInfo().GetGtaRelatedAttribute(a);
                                if (attr.GetKey() == "p_path") {
                                    path = attr.GetValue();
                                }
                                if (attr.GetKey() == "p_deleted") {
                                    notDeleted = !IsTrue(attr.GetValue());
                                }
                            }
                            if (notDeleted) {
                                TFsPath relPath = TFsPath(path).RelativeTo(TFsPath(key)).Fix();
                                auto split = relPath.PathSplit();
                                if (split.size() > 0) {
                                    objects.insert((TString)split[0]);
                                }
                            }
                        }
                    }
                }
            }
            for (auto&& i : objects) {
                result.push_back(i);
            }
            return true;
        } catch (...) {
            ERROR_LOG << "Error on TRTYStorage::GetKeyData: " << CurrentExceptionMessage() << Endl;
        }
        return false;
    }

    bool TRTYStorage::GetKeyData(const TString& pkey, NMetaProtocol::TDocument& doc, i64 version, bool fullData, bool contentOnly) const {
        try {
            TString key = "/" + pkey;
            NMetaProtocol::TReport report;
            TString request = "s_path:\"" + TFsPath(key).Fix().GetPath() + "\" " + (contentOnly ? "s_type:CONTENT" : "(s_type:FILE | s_type:DIR)");
            if (version != -1)
                request = "url:\"" + TFsPath(key).Fix().GetPath() + Sprintf("$$ver%010lu", version) + "\"";
            UrlEscape(request);
            if (SendRequest("text=" + request + "&numdoc=1&how=g_version&fsgta=g_version&fsgta=g_deleted" + (fullData ? "" : "&haha=da"), report)) {
//                DEBUG_LOG << report.DebugString() << Endl;
                if (report.GetGrouping().size() > 0) {
                    const NMetaProtocol::TGrouping& grouping(report.GetGrouping(0));
                    for (int i = 0; i < grouping.GetGroup().size(); i++) {
                        const NMetaProtocol::TGroup& group(grouping.GetGroup(i));
                        for (int d = 0; d < group.GetDocument().size(); d++) {
                            doc = group.GetDocument(d);
                            return true;
                        }
                        return false;
                    }
                }
            }
        } catch (...) {
            ERROR_LOG << "Error on TRTYStorage::GetKeyData: " << CurrentExceptionMessage() << Endl;
        }
        return false;
    }

    bool TRTYStorage::GetVersion(const TString& key, i64& version) const {
        try {
            NMetaProtocol::TDocument doc;
            if (!GetKeyData(key, doc, -1, false))
                return false;
            for (unsigned a = 0; a < doc.FirstStageAttributeSize(); ++a) {
                const NMetaProtocol::TPairBytesBytes& attr = doc.GetFirstStageAttribute(a);
                if (attr.GetKey() == "g_version") {
                    if (!TryFromString(attr.GetValue(), version))
                        return false;
                    return true;
                }
            }
        } catch (...) {
            ERROR_LOG << "Error on TRTYStorage::GetVersion: " << CurrentExceptionMessage() << Endl;
        }
        return false;
    }

    TRTYStorage::TRTYStorage(const IVersionedStorage::TOptions& options)
        : IVersionedStorage(options)
        , IndexingClient(options.RTYStorage.IndexHost, options.RTYStorage.IndexPort, options.RTYStorage.IndexUrl)
    {
        IVersionedStorage::TOptions zooOpt = options;
        zooOpt.Type = IVersionedStorage::TOptions::ZOO;
        ZooStorage.Reset(IVersionedStorage::Create(zooOpt));
    }

}
