#include "helpers.h"
#include "staticserver.h"

#include <saas/searchproxy/configs/searchproxyconfig.h>
#include <saas/searchproxy/core/searchproxyserver.h>

#include <saas/library/daemon_base/config/daemon_config.h>
#include <saas/library/searchmap/parsers/json/json.h>
#include <saas/library/searchmap/searchmap.h>
#include <saas/library/hash_to_block_mode/hash_to_block_mode.h>
#include <saas/api/action.h>

#include <search/idl/meta.pb.h>

#include <kernel/saas_trie/idl/saas_trie.pb.h>
#include <kernel/saas_trie/idl/trie_key.h>

#include <contrib/libs/protobuf/src/google/protobuf/text_format.h>

#include <library/cpp/testing/unittest/registar.h>

#include <util/datetime/base.h>
#include <util/folder/dirut.h>
#include <util/stream/str.h>
#include <util/stream/file.h>
#include <util/string/cast.h>
#include <util/string/subst.h>
#include <util/system/defaults.h>
#include <util/system/maxlen.h>
#include <util/system/sysstat.h>
#include <util/generic/string.h>

using NUnitTest::RandomString;

class TJsonObject;

namespace NSaasTrie {
    TComplexKey MakeKey(const TString& mainKey, const TString& urlMaskPrefix, const TVector<TVector<TString>>& realms) {
        TComplexKey complexKey;
        if (!mainKey.empty() || urlMaskPrefix.empty()) {
            complexKey.SetMainKey(mainKey);
        }
        if (!urlMaskPrefix.empty()) {
            complexKey.SetUrlMaskPrefix(urlMaskPrefix);
        }
        for (auto& realmKeys : realms) {
            Y_ENSURE(realmKeys.size() > 1);
            complexKey.AddKeyRealms(realmKeys[0]);
            auto realm = complexKey.AddAllRealms();
            realm->SetName(realmKeys[0]);
            for (size_t i = 1; i < realmKeys.size(); ++i) {
                realm->AddKey(realmKeys[i]);
            }
        }
        return complexKey;
    }

    const TString QUERY_PREFIX = "/?comp_search=comp:TRIE;max_docs:100;key_type:complex_key_packed&component=TRIE&"
                        "service=trie_service&skip-wizard=1";

    TString MakeQuery(const TString& mainKey, const TString& urlMaskPrefix, const TVector<TVector<TString>>& realms) {
        auto complexKey = MakeKey(mainKey, urlMaskPrefix, realms);
        TString query = QUERY_PREFIX + "&text=" + NSaasTrie::SerializeToCgi(complexKey, true);
        return query;
    }
}

class TSearchProxyTest: public TTestBase {
private:
    UNIT_TEST_SUITE(TSearchProxyTest)
        UNIT_TEST(TestSingleSilentBackend)
        UNIT_TEST(TestSingleSlowpokeBackend)
        UNIT_TEST(TestSingleWithoutData)
        UNIT_TEST(TestSingleWithOneDocument)
        UNIT_TEST(TestSingleWithSilentMaster)
        UNIT_TEST(TestSingleWithSlowpokeMaster)
        UNIT_TEST(TestTwoGroupsWithSingleKps)
        UNIT_TEST(TestTwoGroupsWithKpsInSingleGroup)
        UNIT_TEST(TestTwoGroupsWithKpsInBothGroups)
        UNIT_TEST(TestTwoGroupsWithFourDocuments)
        UNIT_TEST(TestTwoGroupsWithUrlHash)
        UNIT_TEST(TestEncoding)
        UNIT_TEST(TestServiceSharding)
        UNIT_TEST(TestAntiDup)
        UNIT_TEST(TestSearchLogs)
        UNIT_TEST(TestExtraCgi)
        UNIT_TEST(TestMultiReplicGroups)
        UNIT_TEST(TestTrieSharding)
        UNIT_TEST(TestTrieShardingUrlMasks)
        UNIT_TEST(TestTextSplit)
        UNIT_TEST(TestNoTextSplit)
        UNIT_TEST(TestIncorrectTimeout)
        UNIT_TEST(TestGeminiRearrangement)
        UNIT_TEST(TestBlockModeRearrangement)
    UNIT_TEST_SUITE_END();

    static void CheckService(const TString& searchMapConfig, const char* query,
        const char* kps, const TSearchResults& expected, const TString& serviceConfig = "", bool isKv = false)
    {
        TStringInput searchMap(searchMapConfig);
        NSearchMapParser::TJsonSearchMapParser parser(searchMap);
        // Temporary static server will find free port, then occupy and
        // immediately release it.
        const ui16 port = TSilentServer().GetPort();
        TConfigPatcher cp;
        TSearchProxyConfig config(GetPatientSearchProxyConfig(port, "", nullptr, serviceConfig.data()).data(),
            TDaemonConfig(GetDaemonConfig(), true), parser.GetSearchMap(), cp);
        TAutoSearchProxyServer searchProxy(config);
        const TString prefixes[] = {"/test?", "/?service=test&"};
        for (size_t i = 0; i < Y_ARRAY_SIZE(prefixes); ++i) {
            try {
                CheckProtoQuery(port, prefixes[i] + "text=" + query + "&kps=" + kps,
                    expected, isKv);
                CheckJsonQuery(port, prefixes[i] + "text=" + query + "&kps=" + kps,
                    expected);
            } catch (...) {
                Cerr << "While checking query(" << i <<"): " << query << Endl;
                throw;
            }
        }
    }

    static void CheckSingleBackend(ui16 backendPort, const char* query,
        const char* kps, const TSearchResults& expected)
    {
        CheckService(GetSearchMapRule("test", backendPort), query, kps,
            expected);
    }

    static void CheckBackendsPair(ui16 masterPort, ui16 slavePort,
        const char* query, const char* kps, const TSearchResults& expected)
    {
        TVector<TSearchMapLine> lines;
        lines.push_back(GetSearchMapLine("test", slavePort, 0, 65533, "slave"));
        lines.push_back(GetSearchMapLine("test", masterPort));
        CheckService(GetSearchMap(lines), query, kps, expected);
    }

    static void CheckTwoGroups(ui16 firstPort, ui16 secondPort,
        const char* query, const char* kps, const TSearchResults& expected)
    {
        CheckService(GetSearchMap(TVector<TSearchMapLine>
            { GetSearchMapLine("test", firstPort, 0, 32767),
              GetSearchMapLine("test", secondPort, 32768, 65533) }),
            query, kps, expected);
    }

public:
    void TestSingleSilentBackend() {
        TSilentServer backend;
        CheckSingleBackend(backend.GetPort(), "some", "1%2C65533",
            TSearchResults(502));
    }

    void TestSingleSlowpokeBackend() {
        TStaticServer backend(TDuration::MilliSeconds(600));
        const TSearchResults data(200, TSearchResultsGroup(100500,
            TSearchResultsGroup::TUrls(1, RandomString(URL_MAX / 2, 1).Quote())));
        UNIT_ASSERT(backend.AddDoc(
            OneStepRequest(DefaultServiceName, "1%2C65533", "some").data(),
            GenerateReport(data).SerializeAsString()));
        CheckSingleBackend(backend.GetPort(), "some", "1%2C65533", data);
    }

    void TestSingleWithoutData() {
        TStaticServer backend;
        const TSearchResults data(404);
        UNIT_ASSERT(backend.AddDoc(
            OneStepRequest(DefaultServiceName, "1%2C65533", "some").data(),
            GenerateReport(data).SerializeAsString()));
        CheckSingleBackend(backend.GetPort(), "some", "1%2C65533",
            data);
    }

    void TestSingleWithOneDocument() {
        TStaticServer backend;
        const TSearchResults data(200, TSearchResultsGroup(100500,
            TSearchResultsGroup::TUrls(1, RandomString(URL_MAX / 2, 2).Quote())));
        UNIT_ASSERT(backend.AddDoc(
            OneStepRequest(DefaultServiceName, "1%2C65533", "some").data(),
            GenerateReport(data).SerializeAsString()));
        CheckSingleBackend(backend.GetPort(), "some", "1%2C65533", data);
    }

    void TestSingleWithSilentMaster()
    {
        TSilentServer master;
        TStaticServer slave;
        const TSearchResults data(200, TSearchResultsGroup(100500,
            TSearchResultsGroup::TUrls(1, RandomString(URL_MAX / 2, 5).Quote())));
        UNIT_ASSERT(slave.AddDoc(
            OneStepRequest(DefaultServiceName, "1%2C65533", "some").data(),
            GenerateReport(data).SerializeAsString()));
        CheckBackendsPair(master.GetPort(), slave.GetPort(), "some",
            "1%2C65533", data);
    }

    void TestSingleWithSlowpokeMaster() {
        TStaticServer master(TDuration::MilliSeconds(20000));
        TStaticServer slave;
        const TSearchResults masterData(303, TSearchResultsGroup(9000,
            TSearchResultsGroup::TUrls(1, RandomString(URL_MAX / 2, 6).Quote())));
        const TSearchResults slaveData(200, TSearchResultsGroup(100500,
            TSearchResultsGroup::TUrls(1, RandomString(URL_MAX / 2, 7).Quote())));
        UNIT_ASSERT(master.AddDoc(
            OneStepRequest(DefaultServiceName, "1%2C65533", "some").data(),
            GenerateReport(masterData).SerializeAsString()));
        UNIT_ASSERT(slave.AddDoc(
            OneStepRequest(DefaultServiceName, "1%2C65533", "some").data(),
            GenerateReport(slaveData).SerializeAsString()));
        CheckBackendsPair(master.GetPort(), slave.GetPort(), "some",
            "1%2C65533", slaveData);
    }

    void TestTwoGroupsWithSingleKps() {
        TStaticServer first, second;
        const TSearchResults firstData(200, TSearchResultsGroup(9000,
            TSearchResultsGroup::TUrls(1, RandomString(URL_MAX / 2, 10).Quote())));
        const TSearchResults secondData(303, TSearchResultsGroup(100500,
            TSearchResultsGroup::TUrls(1, RandomString(URL_MAX / 2, 11).Quote())));
        UNIT_ASSERT(first.AddDoc(
            OneStepRequest(DefaultServiceName, "1", "some").data(),
            GenerateReport(firstData).SerializeAsString()));
        UNIT_ASSERT(first.AddDoc(
            OneStepRequest(DefaultServiceName, "65532", "some").data(),
            GenerateReport(secondData).SerializeAsString()));
        UNIT_ASSERT(second.AddDoc(
            OneStepRequest(DefaultServiceName, "65532", "some").data(),
            GenerateReport(TSearchResults(200)).SerializeAsString()));
        CheckTwoGroups(first.GetPort(), second.GetPort(), "some",
            "1%2C65532", firstData);
    }

    void TestTwoGroupsWithKpsInSingleGroup() {
        TStaticServer first, second;
        TSearchResults data(200, TSearchResultsGroup(9000,
            TSearchResultsGroup::TUrls(1, RandomString(URL_MAX / 2, 12).Quote())));
        data.push_back(TSearchResultsGroup(100500,
            TSearchResultsGroup::TUrls(1, RandomString(URL_MAX / 2, 13).Quote())));
        UNIT_ASSERT(first.AddDoc(
            OneStepRequest(DefaultServiceName, "1%2C2", "some").data(),
            GenerateReport(data).SerializeAsString()));
        CheckTwoGroups(first.GetPort(), second.GetPort(), "some",
            "1%2C2", data);
    }

    void TestTwoGroupsWithKpsInBothGroups() {
        TStaticServer first, second;
        const TSearchResults firstData(200, TSearchResultsGroup(9000,
            TSearchResultsGroup::TUrls(1, RandomString(URL_MAX / 2, 14).Quote())));
        const TSearchResults secondData(303, TSearchResultsGroup(100500,
            TSearchResultsGroup::TUrls(1, RandomString(URL_MAX / 2, 15).Quote())));
        UNIT_ASSERT(first.AddDoc(
            OneStepRequest(DefaultServiceName, "1", "some").data(),
            GenerateReport(firstData).SerializeAsString()));
        UNIT_ASSERT(second.AddDoc(
            OneStepRequest(DefaultServiceName, "65532", "some").data(),
            GenerateReport(secondData).SerializeAsString()));
        CheckTwoGroups(first.GetPort(), second.GetPort(), "some",
            "1%2C65532", TSearchResults(200, secondData.front(), firstData.front()));
    }

    void TestTwoGroupsWithFourDocuments() {
        TStaticServer first, second;
        const TSearchResults firstData(200, TSearchResultsGroup(9000,
            TSearchResultsGroup::TUrls(1, RandomString(URL_MAX / 2, 16).Quote())));
        const TSearchResults secondData(303, TSearchResultsGroup(100500,
            TSearchResultsGroup::TUrls(1, RandomString(URL_MAX / 2, 17).Quote())));
        UNIT_ASSERT(first.AddDoc(
            OneStepRequest(DefaultServiceName, "1", "some").data(),
            GenerateReport(firstData).SerializeAsString()));
        UNIT_ASSERT(first.AddDoc(
            OneStepRequest(DefaultServiceName, "65532", "some").data(),
            GenerateReport(secondData).SerializeAsString()));
        UNIT_ASSERT(second.AddDoc(
            OneStepRequest(DefaultServiceName, "1", "some").data(),
            GenerateReport(firstData).SerializeAsString()));
        UNIT_ASSERT(second.AddDoc(
            OneStepRequest(DefaultServiceName, "65532", "some").data(),
            GenerateReport(secondData).SerializeAsString()));
        CheckTwoGroups(first.GetPort(), second.GetPort(), "some",
            "1%2C65532", TSearchResults(200, secondData.front(), firstData.front()));
    }

    void TestTwoGroupsWithUrlHash() {
        TStaticServer first, second;
        const TSearchResults firstData(200, TSearchResultsGroup(9000,
            TSearchResultsGroup::TUrls(1, RandomString(URL_MAX / 2, 18).Quote())));
        const TSearchResults secondData(303, TSearchResultsGroup(100500,
            TSearchResultsGroup::TUrls(1, RandomString(URL_MAX / 2, 19).Quote())));
        UNIT_ASSERT(first.AddDoc(
            OneStepRequest(DefaultServiceName, "0%2C1", "some").data(),
            GenerateReport(firstData).SerializeAsString()));
        UNIT_ASSERT(second.AddDoc(
            OneStepRequest(DefaultServiceName, "0%2C1", "some").data(),
            GenerateReport(secondData).SerializeAsString()));
        CheckService(GetSearchMap({
                                      GetSearchMapLine("test", first.GetPort(), 0, 32767, "master", "url_hash"),
                                      GetSearchMapLine("test", second.GetPort(), 32768, 65533, "master", "url_hash")
                                  }),
            "some", "0%2C1", TSearchResults(200, secondData.front(), firstData.front()));
    }

    void TestEncoding() {
        TStaticServer backend;
        const TSearchResults data(200, TSearchResultsGroup(100500,
            TSearchResultsGroup::TUrls(1, "Привет, мир")));
        UNIT_ASSERT(backend.AddDoc(
            OneStepRequest(DefaultServiceName, "1%2C65533", "some").data(),
            GenerateReport(data).SerializeAsString()));
        CheckSingleBackend(backend.GetPort(), "some", "1%2C65533", data);
    }

    void TestServiceSharding() {
        TStaticServer first, second, third;
        const TSearchResults firstData(200,
            TSearchResultsGroup(100500, TSearchResultsGroup::TUrls(1,
                RandomString(URL_MAX / 2, 24).Quote())));
        UNIT_ASSERT(first.AddDoc(
            OneStepRequest("first", "1%2C65533", "some").data(),
            GenerateReport(firstData).SerializeAsString()));
        const TSearchResults secondData(200,
            TSearchResultsGroup(9000, TSearchResultsGroup::TUrls(1,
                RandomString(URL_MAX / 2, 25).Quote())));
        UNIT_ASSERT(second.AddDoc(
            OneStepRequest("second", "1%2C65533", "some").data(),
            GenerateReport(secondData).SerializeAsString()));
        const TSearchResults thirdData(200,
            TSearchResultsGroup(6000, TSearchResultsGroup::TUrls(1,
                RandomString(URL_MAX / 2, 26).Quote())));
        UNIT_ASSERT(third.AddDoc(
            OneStepRequest("third", "1%2C65533", "some").data(),
            GenerateReport(thirdData).SerializeAsString()));
        // request hash sends initial request to the second client
        TVector<TSearchMapLine> lines;
        lines.push_back(GetSearchMapLine("first", first.GetPort()));
        lines.push_back(GetSearchMapLine("second", second.GetPort()));
        lines.push_back(GetSearchMapLine("third", third.GetPort()));
        TString searchMapTxt = GetSearchMap(lines);

        TStringInput searchMap(searchMapTxt);
        NSearchMapParser::TJsonSearchMapParser parser(searchMap);
        // Temporary static server will find free port, then occupy and
        // immediately release it.
        const ui16 port = TSilentServer().GetPort();
        TConfigPatcher cp;
        TSearchProxyConfig config(GetPatientSearchProxyConfig(port).data(),
            TDaemonConfig(GetDaemonConfig(), true), parser.GetSearchMap(), cp);
        TAutoSearchProxyServer searchProxy(config);
        CheckProtoQuery(port,
            "/?service=first&text=some&kps=1%2C65533", firstData);
        CheckProtoQuery(port,
            "/?service=second&text=some&kps=1%2C65533", secondData);
        CheckProtoQuery(port,
            "/?service=third&text=some&kps=1%2C65533", thirdData);
        CheckProtoQuery(port,
            "/?service=first,second&text=some&kps=1%2C65533",
                TSearchResults(200,
                    TSearchResultsGroup(100500, TSearchResultsGroup::TUrls(1,
                        RandomString(URL_MAX / 2, 24).Quote())),
                    TSearchResultsGroup(9000, TSearchResultsGroup::TUrls(1,
                        RandomString(URL_MAX / 2, 25).Quote()))));
    }

    void TestTrieSharding() {
        const ui16 backPort1 = TSilentServer().GetPort();
        TStaticServer backend1(backPort1);
        const ui16 backPort2 = TSilentServer().GetPort();
        TStaticServer backend2(backPort2);
        const TSearchResults data1(200, TSearchResultsGroup(1000, {
            TString("trie_url").Quote()
        }));
        const TSearchResults data2(200, TSearchResultsGroup(1000, {
            TString("trie_url").Quote()
        }));
        const TSearchResults proxyData(200,
            TSearchResultsGroup(1000, {
                TString("trie_url").Quote()
            }),
            TSearchResultsGroup(1000, {
                TString("trie_url").Quote()
            })
        );

        auto backQuery1 = NSaasTrie::MakeQuery("main", {}, {
            {"color", "black"},  // url_hash(mainblack) = 14874
            {"side", "top", "bottom"}
        });
        auto backQuery2 = NSaasTrie::MakeQuery("main", {}, {
            {"color", "white"},  // url_hash(mainwhite) = 54669
            {"side", "top", "bottom"}
        });
        auto proxyQuery = NSaasTrie::MakeQuery("main", {}, {
            {"color", "white", "black"},
            {"side", "top", "bottom"}
        }) + "&sp_meta_search=multi_proxy";

        backend1.SetKV(true);
        backend2.SetKV(true);
        UNIT_ASSERT(backend1.AddDoc(backQuery1.data(), GenerateReport(data1, true).SerializeAsString()));
        UNIT_ASSERT(backend2.AddDoc(backQuery2.data(), GenerateReport(data2, true).SerializeAsString()));

        TVector<TSearchMapLine> lines;
        lines.emplace_back(GetSearchMapLine("trie_service", backend1.GetPort(), 0, 32766, "rank", "url_hash"));
        lines.emplace_back(GetSearchMapLine("trie_service", backend2.GetPort(), 32767, 65533, "rank", "url_hash"));
        TString searchMapTxt = GetSearchMap(lines);

        TStringInput searchMap(searchMapTxt);
        NSearchMapParser::TJsonSearchMapParser parser(searchMap);

        const TString serviceConfig = "<Service>\n"
                "Name: trie_service\n"
                "TwoStepQuery: false\n"
                "GroupingByDC: false\n"
                "<ProxyMeta>\n"
                    "ParallelRequestCount: 2\n"
                    "MaxAttempts: 2\n"
                "</ProxyMeta>\n"
                "<CgiParams>\n"
                    "meta_search: first_found\n"
                    "normal_kv_report: da\n"
                "</CgiParams>\n"
                "<CustomRearranges>\n"
                    "TrieSharding: empty\n"
                "</CustomRearranges>\n"
            "</Service>";

        const ui16 searchProxyPort = TSilentServer().GetPort();
        TConfigPatcher cp;
        TSearchProxyConfig config(GetPatientSearchProxyConfig(searchProxyPort, "", nullptr, serviceConfig.data()).data(),
            TDaemonConfig(GetDaemonConfig(), true), parser.GetSearchMap(), cp);
        TAutoSearchProxyServer searchProxy(config);
        CheckProtoQuery(searchProxyPort, proxyQuery, proxyData, true);
        {
            auto complexKey1 = NSaasTrie::MakeKey("main", {}, {
                {"color", "black"},  // url_hash(mainblack) = 14874
                {"side", "top", "bottom"}
            });

            auto complexKey2 = NSaasTrie::MakeKey("main", {}, {
                {"color", "white"},  // url_hash(mainblack) = 14874
                {"side", "top", "bottom"}
            });
            TString multiQuery = NSaasTrie::QUERY_PREFIX +
                "&text=" + NSaasTrie::SerializeToCgi(complexKey1, true) +
                "&text=" + NSaasTrie::SerializeToCgi(complexKey2, true) +
                "&sp_meta_search=multi_proxy";
            CheckProtoQuery(searchProxyPort, multiQuery, proxyData, true);
        }
    }

    void TestTrieShardingUrlMasks() {
        const ui16 backPort1 = TSilentServer().GetPort();
        TStaticServer backend1(backPort1);
        const ui16 backPort2 = TSilentServer().GetPort();
        TStaticServer backend2(backPort2);
        const TSearchResults data1(200, TSearchResultsGroup(1000, {
            TString("trie_url").Quote()
        }));
        const TSearchResults data2(200, TSearchResultsGroup(1000, {
            TString("trie_url").Quote()
        }));
        const TSearchResults proxyData(200,
            TSearchResultsGroup(1000, {
                TString("trie_url").Quote()
            }),
            TSearchResultsGroup(1000, {
                TString("trie_url").Quote()
            })
        );

        auto backQuery1 = NSaasTrie::MakeQuery({}, ".9", {
            {"7", "\thttp://subdomain.example.com/page.html"}
            // url_hash(.9\texample.com) = 9031
        });
        auto backQuery2 = NSaasTrie::MakeQuery(".7", ".9", {
            {"7", "\thttp://subdomain.example.com/page.html"}
            // url_hash(.7\thttp://subdomain.example.com/page.html) = 39518
            // url_hash(.9\tsubdomain.example.com) = 64589
        });
        auto proxyQuery = NSaasTrie::MakeQuery(".7", ".9", {
            {"7", "\thttp://subdomain.example.com/page.html"}
        }) + "&sp_meta_search=multi_proxy";

        backend1.SetKV(true);
        backend2.SetKV(true);
        UNIT_ASSERT(backend1.AddDoc(backQuery1.data(), GenerateReport(data1, true).SerializeAsString()));
        UNIT_ASSERT(backend2.AddDoc(backQuery2.data(), GenerateReport(data2, true).SerializeAsString()));

        TVector<TSearchMapLine> lines;
        lines.emplace_back(GetSearchMapLine("trie_service", backend1.GetPort(), 0, 32766, "rank", "url_hash"));
        lines.emplace_back(GetSearchMapLine("trie_service", backend2.GetPort(), 32767, 65533, "rank", "url_hash"));
        TString searchMapTxt = GetSearchMap(lines);

        TStringInput searchMap(searchMapTxt);
        NSearchMapParser::TJsonSearchMapParser parser(searchMap);

        const TString serviceConfig = "<Service>\n"
                "Name: trie_service\n"
                "TwoStepQuery: false\n"
                "GroupingByDC: false\n"
                "<ProxyMeta>\n"
                    "ParallelRequestCount: 2\n"
                    "MaxAttempts: 2\n"
                "</ProxyMeta>\n"
                "<CgiParams>\n"
                    "meta_search: first_found\n"
                    "normal_kv_report: da\n"
                "</CgiParams>\n"
                "<CustomRearranges>\n"
                    "TrieSharding: empty\n"
                "</CustomRearranges>\n"
            "</Service>";

        const ui16 searchProxyPort = TSilentServer().GetPort();
        TConfigPatcher cp;
        TSearchProxyConfig config(GetPatientSearchProxyConfig(searchProxyPort, "", nullptr, serviceConfig.data()).data(),
            TDaemonConfig(GetDaemonConfig(), true), parser.GetSearchMap(), cp);
        TAutoSearchProxyServer searchProxy(config);
        CheckProtoQuery(searchProxyPort, proxyQuery, proxyData, true);
    }

    void TestAntiDup() {
        TStaticServer first, second, third;
        const TSearchResults firstData(200,
            TSearchResultsGroup(100500, TSearchResultsGroup::TUrls(1,
                RandomString(URL_MAX / 2, 26).Quote())));
        UNIT_ASSERT(first.AddDoc(
            OneStepRequest("first", "1%2C65533", "some").data(),
            GenerateReport(firstData).SerializeAsString()));
        const TSearchResults secondData(200,
            TSearchResultsGroup(9000, TSearchResultsGroup::TUrls(1,
                RandomString(URL_MAX / 2, 27).Quote())));
        UNIT_ASSERT(second.AddDoc(
            OneStepRequest("second", "1%2C65533", "some").data(),
            GenerateReport(secondData).SerializeAsString()));
        UNIT_ASSERT(third.AddDoc(
            OneStepRequest("third", "1%2C65533", "some").data(),
            GenerateReport(secondData).SerializeAsString()));
        // request hash sends initial request to the second client
        TVector<TSearchMapLine> lines;
        lines.push_back(GetSearchMapLine("first", first.GetPort()));
        lines.push_back(GetSearchMapLine("second", second.GetPort()));
        lines.push_back(GetSearchMapLine("third", third.GetPort()));
        TString searchMapTxt = GetSearchMap(lines);

        TStringInput searchMap(searchMapTxt);
        NSearchMapParser::TJsonSearchMapParser parser(searchMap);
        // Temporary static server will find free port, then occupy and
        // immediately release it.
        const ui16 port = TSilentServer().GetPort();
        TConfigPatcher cp;
        TSearchProxyConfig config(
            GetPatientSearchProxyConfig(port, "ReArrangeOptions: AntiDup").data(),
            TDaemonConfig(GetDaemonConfig(), true), parser.GetSearchMap(), cp);
        TAutoSearchProxyServer searchProxy(config);
        CheckProtoQuery(port,
            "/?service=first&text=some&kps=1%2C65533", firstData);
        CheckProtoQuery(port,
            "/?service=second&text=some&kps=1%2C65533", secondData);
        CheckProtoQuery(port,
            "/?service=third&text=some&kps=1%2C65533", secondData);
        CheckProtoQuery(port,
            "/?service=first,second,third&text=some&kps=1%2C65533",
                TSearchResults(200,
                    TSearchResultsGroup(100500, TSearchResultsGroup::TUrls(1,
                        RandomString(URL_MAX / 2, 26).Quote())),
                    TSearchResultsGroup(9000, TSearchResultsGroup::TUrls(1,
                        RandomString(URL_MAX / 2, 27).Quote()))));
    }

    class TUniqueKvp: public TKeyValueParser<TUniqueKvp> {
    public:
        TUniqueKvp(const TString& s) {
            if (!DoParse(s, '\t'))
                ythrow yexception() << "Cannot parse: " << Errors();
        }

        bool TreatKeyValue(const TStringBuf key, const TStringBuf value) {
            return Values.insert(std::make_pair(key, value)).second;
        }

        THashMap<TStringBuf, TStringBuf> Values;
    };

    void TestSearchLogs() {
        TStaticServer backend;
        const TSearchResults firstData(200, TSearchResultsGroup(100500, TSearchResultsGroup::TUrls(1,
                                       RandomString(URL_MAX / 2, 26).Quote())));
        UNIT_ASSERT(backend.AddDoc(OneStepRequest("tests", "1", "yandex").data(), GenerateReport(firstData).SerializeAsString()));
        Mkdir("./splogs", MODE0755);

        const TString searchMapText = GetSearchMapRule("tests", backend.GetPort());
        TStringInput searchMapStream(searchMapText);
        NSearchMapParser::TJsonSearchMapParser parser(searchMapStream);

        const ui16 port = TSilentServer().GetPort();
        TConfigPatcher cp;
        TSearchProxyConfig config(
            GetPatientSearchProxyConfig(port, "", "./splogs").data(),
            TDaemonConfig(GetDaemonConfig(), true), parser.GetSearchMap(), cp);
        TAutoSearchProxyServer searchProxy(config);

        THashMap<TString, TString> extraHeaders;
        extraHeaders["X-Yandex-ICookie"] = "1234";
        const THttpResponse& response = GetHttpData(port, "/?service=tests&text=yandex&kps=1&lr=225&parent-reqid=12345&ms=proto", extraHeaders);
        UNIT_ASSERT(response.HttpCode == 200);
        searchProxy.ReopenLog();
        const TString& workDir = config.GetServiceConfig("").GetMetaSearchConfig()->WorkDir;
        TString accessFile = "./splogs/access.log";
        TString reqansFile = "./splogs/reqans.log";
        resolvepath(accessFile, workDir);
        resolvepath(reqansFile, workDir);
        const TString accessString = TUnbufferedFileInput(accessFile).ReadAll();
        const TString reqansString = TUnbufferedFileInput(reqansFile).ReadAll();
        RemoveDirWithContents("./splogs");

        TUniqueKvp access(accessString);
        TUniqueKvp reqans(reqansString);

        if (!IsTrue(access.Values["fake-uid"]))
            Cerr << "will fail now: " << accessString << Endl;
        UNIT_ASSERT(IsTrue(access.Values["fake-uid"]));
        UNIT_ASSERT(access.Values["uid"].StartsWith("saas_fake"));
        UNIT_ASSERT_VALUES_EQUAL(access.Values["kps"], "1");
        UNIT_ASSERT_VALUES_EQUAL(access.Values["service"], "tests");
        UNIT_ASSERT_VALUES_EQUAL(access.Values["query"], "yandex");
        UNIT_ASSERT_VALUES_EQUAL(access.Values["icookie"], "1234");
        UNIT_ASSERT_VALUES_UNEQUAL(access.Values["ts"], "");

        UNIT_ASSERT(IsTrue(reqans.Values["fake-uid"]));
        UNIT_ASSERT(reqans.Values["uid"].StartsWith("saas_fake"));
        UNIT_ASSERT_VALUES_EQUAL(reqans.Values["kps"], "1");
        UNIT_ASSERT_VALUES_EQUAL(reqans.Values["service"], "tests");
        UNIT_ASSERT_VALUES_EQUAL(reqans.Values["query"], "yandex");
        UNIT_ASSERT_VALUES_EQUAL(reqans.Values["icookie"], "1234");
        UNIT_ASSERT_VALUES_UNEQUAL(reqans.Values["ts"], "");
        UNIT_ASSERT_VALUES_EQUAL(reqans.Values["user-region"], "225");
        UNIT_ASSERT_VALUES_EQUAL(reqans.Values["parent-reqid"], "12345");
    }

    void TestExtraCgi() {
        TConfigPatcher cp;
        TServiceConfig serviceConfig;

        const TString serviceConfigString = "<Service>\n"
            "<GlobalCgiParams>\n"
                "$service : ((service == \"maps_graph_russia\" and string.match(text, \"car_selection\")) and \"taxi_car_selection\" or service)\n"
                "+gta : time&space\n"
                "-relev : attr_limit=999\n"
                "pron : pruncount40\n"
                "?meta_search : first_found\n"
                "$optional: expr and 'set' or ''\n"
            "</GlobalCgiParams>\n"
        "</Service>";
        TAnyYandexConfig yandexConfig;
        UNIT_ASSERT(yandexConfig.ParseMemory(serviceConfigString.data()));

        serviceConfig.InitFromSection(yandexConfig.GetRootSection()->Child, cp);
        auto corrector = MakeHolder<TServiceGlobalCgiCorrector>(serviceConfig);
        {
            TCgiParameters cgi;
            cgi.Scan("gta=yandex&relev=nofml&relev=attr_limit=999&pron=termsearch&meta_search=none&service=maps_graph_russia&text=type:route");
            corrector->FormCgi(cgi, nullptr);
            UNIT_ASSERT_VALUES_EQUAL(cgi.Print(), "gta=yandex&gta=time&gta=space&meta_search=none&pron=pruncount40&relev=nofml&service=maps_graph_russia&text=type%3Aroute");
        }
        {
            TCgiParameters cgi;
            cgi.Scan("service=maps_graph_russia&text=type:car_selection");
            corrector->FormCgi(cgi, nullptr);
            UNIT_ASSERT_VALUES_EQUAL(cgi.Print(), "gta=time&gta=space&meta_search=first_found&pron=pruncount40&service=taxi_car_selection&text=type%3Acar_selection");
        }
        {
            TCgiParameters cgi;
            cgi.Scan("service=maps_graph_russia&text=type:car_selection&expr=");
            corrector->FormCgi(cgi, nullptr);
            UNIT_ASSERT_VALUES_EQUAL(cgi.Print(), "expr=&gta=time&gta=space&meta_search=first_found&optional=set&pron=pruncount40&service=taxi_car_selection&text=type%3Acar_selection");
        }
    }

    TVector<ui32> RunMultiReplicGroups(const std::initializer_list<TString>& replicGroups, const ui32 parallelRequestCount, const bool groupingByDC) {
        TVector<TStaticServer> servers(replicGroups.size());
        const TSearchResults data(200, TSearchResultsGroup(9000,
            TSearchResultsGroup::TUrls(1, RandomString(URL_MAX / 2, 18).Quote())));

        TVector<TSearchMapLine> lines;
        lines.reserve(servers.size());
        ui32 idx = 0;
        for (auto& grp: replicGroups) {
            auto& server = servers[idx++];
            server.SetKV();
            UNIT_ASSERT(server.AddDoc(
                KvRequest(DefaultServiceName, "1", "some").data(),
                GenerateReport(data, true).SerializeAsString()));
            lines.push_back(GetSearchMapLine("test", server.GetPort(), 0, 65533, "master", "url_hash"));
            lines.back().Group = grp;
        }

        const TString grpByDC = groupingByDC ? "" : "GroupingByDC: false\n";
        const TString serviceConfig = "<Service>\n"
                "Name: test\n"
                + grpByDC +
                "<ProxyMeta>\n"
                    "ParallelRequestCount: " + ToString(parallelRequestCount) + "\n"
                    "MaxAttempts: 2\n"
                "</ProxyMeta>\n"
                "<CgiParams>\n"
                    "meta_search: first_found\n"
                    "normal_kv_report: da\n"
                "</CgiParams>\n"
            "</Service>";

        CheckService(GetSearchMap(lines),
            "some&sp_meta_search=proxy", "1", TSearchResults(200, data.front()), serviceConfig, true);

        TVector<ui32> res;
        res.reserve(servers.size());
        ui32 requestCount = 0;
        for (const auto& server: servers) {
            res.push_back(server.GetCallsCount());
            requestCount += server.GetCallsCount();
        }
        UNIT_ASSERT(requestCount <= 4 * parallelRequestCount);
        UNIT_ASSERT(requestCount >= 4);
        return res;
    }

    void TestMultiReplicGroups() {
        RunMultiReplicGroups({"dc1-1@1@0@0@1", "dc1-1@1@0@0@1", "dc2-0@0@1@1@1", "dc2-0@0@1@1@1"}, 2, true );
    }

    void TestTextSplit() {
        const ui16 backPort1 = TSilentServer().GetPort();
        TStaticServer backend1(backPort1);
        const ui16 backPort2 = TSilentServer().GetPort();
        TStaticServer backend2(backPort2);
        const TSearchResults data1(200, TSearchResultsGroup(1000, {
            TString("test_url").Quote()
        }));
        const TSearchResults data2(200, TSearchResultsGroup(1000, {
            TString("test_url").Quote()
        }));
        const TSearchResults proxyData(200,
            TSearchResultsGroup(1000, {
                TString("test_url").Quote()
            }),
            TSearchResultsGroup(1000, {
                TString("test_url").Quote()
            })
        );

        TString backQuery1 = "/?service=test_service&text=white&sgkps=1,2";
        TString backQuery2 = "/?service=test_service&text=black&sgkps=1,2";
        TString proxyQuery = "/?service=test_service&text=white,black&sgkps=1,2&sp_meta_search=multi_proxy";

        backend1.SetKV(true);
        backend2.SetKV(true);
        UNIT_ASSERT(backend1.AddDoc(backQuery1.data(), GenerateReport(data1, true).SerializeAsString()));
        UNIT_ASSERT(backend2.AddDoc(backQuery2.data(), GenerateReport(data2, true).SerializeAsString()));

        TVector<TSearchMapLine> lines;
        lines.emplace_back(GetSearchMapLine("test_service", backend1.GetPort(), 0, 32766, "rank", "url_hash"));
        lines.emplace_back(GetSearchMapLine("test_service", backend2.GetPort(), 32767, 65533, "rank", "url_hash"));
        TString searchMapTxt = GetSearchMap(lines);

        TStringInput searchMap(searchMapTxt);
        NSearchMapParser::TJsonSearchMapParser parser(searchMap);

        const TString serviceConfig = "<Service>\n"
                "Name: test_service\n"
                "TwoStepQuery: false\n"
                "GroupingByDC: false\n"
                "<ProxyMeta>\n"
                    "ParallelRequestCount: 2\n"
                    "MaxAttempts: 2\n"
                "</ProxyMeta>\n"
                "<CgiParams>\n"
                    "meta_search: first_found\n"
                    "normal_kv_report: da\n"
                "</CgiParams>\n"
            "</Service>";

        const ui16 searchProxyPort = TSilentServer().GetPort();
        TConfigPatcher cp;
        TSearchProxyConfig config(GetPatientSearchProxyConfig(searchProxyPort, "", nullptr, serviceConfig.data()).data(),
            TDaemonConfig(GetDaemonConfig(), true), parser.GetSearchMap(), cp);
        TAutoSearchProxyServer searchProxy(config);
        CheckProtoQuery(searchProxyPort, proxyQuery, proxyData, true);
    }

    void TestNoTextSplit() {
        const ui16 backPort1 = TSilentServer().GetPort();
        TStaticServer backend1(backPort1);
        const ui16 backPort2 = TSilentServer().GetPort();
        TStaticServer backend2(backPort2);
        const TSearchResults data1(200, TSearchResultsGroup(1000, {
            TString("test_url").Quote()
        }));
        const TSearchResults data2(200, TSearchResultsGroup(1000, {
            TString("test_url").Quote()
        }));
        const TSearchResults proxyData(200,
            TSearchResultsGroup(1000, {
                TString("test_url").Quote()
            }),
            TSearchResultsGroup(1000, {
                TString("test_url").Quote()
            })
        );

        TString backQuery1 = "/?service=test_service&saas_no_text_split=1&text=color,black&text=color,red";
        TString backQuery2 = "/?service=test_service&saas_no_text_split=1&text=color,wwhite";
        TString proxyQuery = "/?service=test_service&text=color,black&text=color,wwhite&text=color,red&sp_meta_search=multi_proxy";

        backend1.SetKV(true);
        backend2.SetKV(true);
        UNIT_ASSERT(backend1.AddDoc(backQuery1.data(), GenerateReport(data1, true).SerializeAsString()));
        UNIT_ASSERT(backend2.AddDoc(backQuery2.data(), GenerateReport(data2, true).SerializeAsString()));

        TVector<TSearchMapLine> lines;
        lines.emplace_back(GetSearchMapLine("test_service", backend1.GetPort(), 0, 32766, "rank", "url_hash"));
        lines.emplace_back(GetSearchMapLine("test_service", backend2.GetPort(), 32767, 65533, "rank", "url_hash"));
        TString searchMapTxt = GetSearchMap(lines);

        TStringInput searchMap(searchMapTxt);
        NSearchMapParser::TJsonSearchMapParser parser(searchMap);

        const TString serviceConfig = "<Service>\n"
                "Name: test_service\n"
                "TwoStepQuery: false\n"
                "GroupingByDC: false\n"
                "<ProxyMeta>\n"
                    "ParallelRequestCount: 2\n"
                    "MaxAttempts: 2\n"
                "</ProxyMeta>\n"
                "<GlobalCgiParams>\n"
                    "meta_search: first_found\n"
                    "normal_kv_report: da\n"
                    "saas_no_text_split: 1\n"
                "</GlobalCgiParams>\n"
            "</Service>";

        const ui16 searchProxyPort = TSilentServer().GetPort();
        TConfigPatcher cp;
        TSearchProxyConfig config(GetPatientSearchProxyConfig(searchProxyPort, "", nullptr, serviceConfig.data()).data(),
            TDaemonConfig(GetDaemonConfig(), true), parser.GetSearchMap(), cp);
        TAutoSearchProxyServer searchProxy(config);
        CheckProtoQuery(searchProxyPort, proxyQuery, proxyData, true);
    }

    void TestNoTextSplitNoText() {
        const ui16 backPort1 = TSilentServer().GetPort();
        TStaticServer backend1(backPort1);
        TString proxyQuery = "/?service=test_service&sp_meta_search=multi_proxy";
        TString backQuery1 = "/?service=test_service&text=test_url";
        const TSearchResults data1(200, TSearchResultsGroup(1000, {
            TString("test_url").Quote()
        }));

        const TSearchResults proxyDataValid(200,
            TSearchResultsGroup(1000, {
                TString("test_url").Quote()
            })
        );

        const TSearchResults proxyDataInValid(400);

        backend1.SetKV(true);
        UNIT_ASSERT(backend1.AddDoc(backQuery1.data(), GenerateReport(data1, true).SerializeAsString()));

        TVector<TSearchMapLine> lines;
        lines.emplace_back(GetSearchMapLine("test_service", backend1.GetPort(), 0, 65533, "rank", "url_hash"));
        TString searchMapTxt = GetSearchMap(lines);

        TStringInput searchMap(searchMapTxt);
        NSearchMapParser::TJsonSearchMapParser parser(searchMap);

        const TString serviceConfig = "<Service>\n"
                "Name: test_service\n"
                "TwoStepQuery: false\n"
                "GroupingByDC: false\n"
                "<ProxyMeta>\n"
                    "ParallelRequestCount: 2\n"
                    "MaxAttempts: 2\n"
                "</ProxyMeta>\n"
                "<GlobalCgiParams>\n"
                    "meta_search: first_found\n"
                    "normal_kv_report: da\n"
                "</GlobalCgiParams>\n"
            "</Service>";

        const ui16 searchProxyPort = TSilentServer().GetPort();
        TConfigPatcher cp;
        TSearchProxyConfig config(GetPatientSearchProxyConfig(searchProxyPort, "", nullptr, serviceConfig.data()).data(),
            TDaemonConfig(GetDaemonConfig(), true), parser.GetSearchMap(), cp);
        TAutoSearchProxyServer searchProxy(config);
        CheckProtoQuery(searchProxyPort, proxyQuery + "&text=test_url", proxyDataValid, true);
        CheckProtoQuery(searchProxyPort, proxyQuery, proxyDataInValid, true);
    }

    void TestIncorrectTimeout() {
        const ui16 backPort1 = TSilentServer().GetPort();
        TStaticServer backend1(backPort1);
        TString proxyQuery = "/?service=test_service&sp_meta_search=multi_proxy";
        TString backQuery1 = "/?service=test_service&text=test_url";
        const TSearchResults data1(200, TSearchResultsGroup(1000, {
            TString("test_url").Quote()
        }));

        const TSearchResults proxyDataInValid(400);

        backend1.SetKV(true);
        UNIT_ASSERT(backend1.AddDoc(backQuery1.data(), GenerateReport(data1, true).SerializeAsString()));

        TVector<TSearchMapLine> lines;
        lines.emplace_back(GetSearchMapLine("test_service", backend1.GetPort(), 0, 65533, "rank", "url_hash"));
        TString searchMapTxt = GetSearchMap(lines);

        TStringInput searchMap(searchMapTxt);
        NSearchMapParser::TJsonSearchMapParser parser(searchMap);

        const TString serviceConfig = "<Service>\n"
                "Name: test_service\n"
                "TwoStepQuery: false\n"
                "GroupingByDC: false\n"
                "<ProxyMeta>\n"
                    "ParallelRequestCount: 2\n"
                    "MaxAttempts: 2\n"
                "</ProxyMeta>\n"
                "<GlobalCgiParams>\n"
                    "meta_search: first_found\n"
                    "normal_kv_report: da\n"
                "</GlobalCgiParams>\n"
            "</Service>";

        const ui16 searchProxyPort = TSilentServer().GetPort();
        TConfigPatcher cp;
        TSearchProxyConfig config(GetPatientSearchProxyConfig(searchProxyPort, "", nullptr, serviceConfig.data()).data(),
            TDaemonConfig(GetDaemonConfig(), true), parser.GetSearchMap(), cp);
        TAutoSearchProxyServer searchProxy(config);
        CheckProtoQuery(searchProxyPort, proxyQuery + "&timeout=0", proxyDataInValid, true);
    }

    void TestGeminiRearrangement() {
        const ui16 backPort1 = TSilentServer().GetPort();
        TStaticServer backend1(backPort1);

        // Fill backend with data
        backend1.SetKV(true);
        TString successfulQuery = "/?gemini_type=weak&saas_no_text_split=yes&service=test_service&sgkps=0%2C1&text=https%3A//yandex.ru/&text=http%3A//yandex.ru/";
        UNIT_ASSERT(backend1.AddDoc(successfulQuery.data(), GenerateGeminiReport("https://yandex.ru/", "https://yandex.ru/").SerializeAsString()));

        TString noResultsQuery = "/?gemini_type=weak&saas_no_text_split=yes&service=test_service&sgkps=0%2C1&text=https%3A//myandex.ru/";
        UNIT_ASSERT(backend1.AddDoc(noResultsQuery.data(), GenerateEmptyReport().SerializeAsString()));

        // Prepare searchmap
        TVector<TSearchMapLine> lines;
        lines.emplace_back(GetSearchMapLine("test_service", backend1.GetPort(), 0, 65533, "rank", "url_hash"));
        TString searchMapTxt = GetSearchMap(lines);

        TStringInput searchMap(searchMapTxt);
        NSearchMapParser::TJsonSearchMapParser parser(searchMap);

        // Initialize searchproxy
        const TString serviceConfig = "<Service>\n"
                "Name: test_service\n"
                "TwoStepQuery: false\n"
                "GroupingByDC: false\n"
                "<ProxyMeta>\n"
                    "ParallelRequestCount: 2\n"
                    "MaxAttempts: 2\n"
                "</ProxyMeta>\n"
                "<GlobalCgiParams>\n"
                    "meta_search: first_found\n"
                    "normal_kv_report: da\n"
                    "sp_meta_search: multi_proxy\n"
                    "saas_no_text_split: yes\n"
                "</GlobalCgiParams>\n"
                "<CustomRearranges>\n"
                "    Gemini: RootDir=data\n"
                "</CustomRearranges>\n"
            "</Service>";

        const ui16 searchProxyPort = TSilentServer().GetPort();
        TConfigPatcher cp;
        TSearchProxyConfig config(GetPatientSearchProxyConfig(searchProxyPort, "", nullptr, serviceConfig.data()).data(),
            TDaemonConfig(GetDaemonConfig(), true), parser.GetSearchMap(), cp);

        TAutoSearchProxyServer searchProxy(config);

        // Test successful request (WEAK type)
        {
            TString proxyQuery = "/?service=test_service&sp_meta_search=multi_proxy&text=http%3A%2F%2Fyandex.ru&gemini_type=weak";

            NGeminiProtos::TCastorResponse expected;
            expected.SetOriginalUrl("http://yandex.ru");
            expected.SetCanonizedUrl("https://yandex.ru/");
            expected.SetCanonizationType(NGeminiProtos::ECanonizationType::WEAK);
            expected.AddMainUrl("https://yandex.ru/");

            CheckGeminiQuery(searchProxyPort, proxyQuery, expected);
        }

        //Test successful request (CANONIZE type)
        {
            TString proxyQuery = "/?service=test_service&sp_meta_search=multi_proxy&text=http%3A%2F%2Fyandex.ru&gemini_type=canonize_weak";

            NGeminiProtos::TCastorResponse expected;
            expected.SetOriginalUrl("http://yandex.ru");
            expected.SetCanonizedUrl("https://yandex.ru/");
            expected.SetCanonizationType(NGeminiProtos::ECanonizationType::CANONIZE_WEAK);

            CheckGeminiQuery(searchProxyPort, proxyQuery, expected);
        }

        //Test failed request (unknown canonization type)
        {
            TString proxyQuery = "/?service=test_service&sp_meta_search=multi_proxy&text=http%3A%2F%2Fyandex.ru&gemini_type=badtype";

            NGeminiProtos::TCastorResponse expected;
            expected.SetOriginalUrl("http://yandex.ru");
            expected.SetError(NGeminiProtos::EErrorType::UNKNOWN_CANONIZATION_TYPE);

            CheckGeminiQuery(searchProxyPort, proxyQuery, expected);
        }

        //Test failed request (no urls)
        {
            TString proxyQuery = "/?service=test_service&sp_meta_search=multi_proxy&gemini_type=canonize";

            NGeminiProtos::TCastorResponse expected;
            CheckGeminiQuery(searchProxyPort, proxyQuery, expected);
        }

        //Test failed request (bad url)
        {
            TString proxyQuery = "/?service=test_service&sp_meta_search=multi_proxy&text=somebadurl&gemini_type=canonize_weak";

            NGeminiProtos::TCastorResponse expected;
            expected.SetOriginalUrl("somebadurl");
            expected.SetCanonizationType(NGeminiProtos::ECanonizationType::CANONIZE_WEAK);
            expected.SetError(NGeminiProtos::EErrorType::BAD_URL);

            CheckGeminiQuery(searchProxyPort, proxyQuery, expected);
        }

        //Test failed request (url not found)
        {
            TString proxyQuery = "/?service=test_service&sp_meta_search=multi_proxy&text=https%3A%2F%2Fmyandex.ru&gemini_type=weak";

            NGeminiProtos::TCastorResponse expected;
            expected.SetCanonizationType(NGeminiProtos::ECanonizationType::WEAK);
            expected.SetOriginalUrl("https://myandex.ru");
            expected.SetCanonizedUrl("https://myandex.ru/");
            expected.SetError(NGeminiProtos::EErrorType::URL_NOT_FOUND);

            CheckGeminiQuery(searchProxyPort, proxyQuery, expected);
        }
    }

    void TestBlockModeRearrangement() {
        const ui16 backPort1 = TSilentServer().GetPort();
        TStaticServer backend1(backPort1);

        // Fill backend with data
        backend1.SetKV(true);
        TBlockHash blockHash(1);
        TString blockKey = ToString(blockHash.GetBlockKey("kps", "url_1"));
        TString propKey1 = blockHash.GetGtaKey("kps", "url_1");
        TString propKey2 = blockHash.GetGtaKey("kps", "url_2");

        TString successfulQuery = "/?service=test_service&text=" + blockKey;

        // Made report
        NMetaProtocol::TReport report;
        report.MutableDebugInfo();
        NMetaProtocol::TGrouping* grouping = report.AddGrouping();
        for (size_t i = 0; i < 3; i++) {
            report.AddTotalDocCount(1);
        }
        NMetaProtocol::TGroup* group = grouping->AddGroup();
        NMetaProtocol::TDocument* doc = group->AddDocument();
        doc->SetUrl(blockKey);

        auto* attr = doc->MutableArchiveInfo()->AddGtaRelatedAttribute();
        attr->SetKey(propKey1);
        NSaas::TAction action1;
        NSaas::TDocument& doc1 = action1.AddDocument();
        doc1.SetUrl("url_1");
        doc1.AddProperty("gta_1", "1");
        attr->SetValue(action1.ToProtobuf().GetDocument().SerializeAsString());

        attr = doc->MutableArchiveInfo()->AddGtaRelatedAttribute();
        attr->SetKey(propKey2);
        NSaas::TAction action2;
        NSaas::TDocument& doc2 = action2.AddDocument();
        doc2.SetUrl("url_2");
        doc2.AddProperty("gta_2", "2");
        attr->SetValue(action2.ToProtobuf().GetDocument().SerializeAsString());

        UNIT_ASSERT(backend1.AddDoc(successfulQuery.data(), report.SerializeAsString()));

        // Prepare searchmap
        TVector<TSearchMapLine> lines;
        lines.emplace_back(GetSearchMapLine("test_service", backend1.GetPort(), 0, 65533, "rank", "url_hash"));
        TString searchMapTxt = GetSearchMap(lines);

        TStringInput searchMap(searchMapTxt);
        NSearchMapParser::TJsonSearchMapParser parser(searchMap);

        // Initialize searchproxy
        const TString serviceConfig = "<Service>\n"
                "Name: test_service\n"
                "TwoStepQuery: false\n"
                "GroupingByDC: false\n"
                "<ProxyMeta>\n"
                    "ParallelRequestCount: 2\n"
                    "MaxAttempts: 2\n"
                "</ProxyMeta>\n"
                "<GlobalCgiParams>\n"
                    "meta_search: first_found\n"
                    "normal_kv_report: da\n"
                "</GlobalCgiParams>\n"
                "<CustomRearranges>\n"
                "    BlockMode: TotalDocCount=2;MaxBlockDocCount=2\n"
                "</CustomRearranges>\n"
            "</Service>";

        const ui16 searchProxyPort = TSilentServer().GetPort();
        TConfigPatcher cp;
        TSearchProxyConfig config(GetPatientSearchProxyConfig(searchProxyPort, "", nullptr, serviceConfig.data()).data(),
            TDaemonConfig(GetDaemonConfig(), true), parser.GetSearchMap(), cp);

        TAutoSearchProxyServer searchProxy(config);

        {
            TString proxyQuery = "/?service=test_service&sp_meta_search=proxy&text=url_1&sgkps=kps&ms=proto&hr=da&dump=eventlog";
            NMetaProtocol::TReport answer;
            const THttpResponse& response = GetHttpData(searchProxyPort, proxyQuery.data());
            ::google::protobuf::TextFormat::ParseFromString(response.Body, &answer);
            UNIT_ASSERT(answer.MutableGrouping(0)->MutableGroup(0)->MutableDocument(0)->GetUrl() == "url_1");
            UNIT_ASSERT(answer.MutableGrouping(0)->MutableGroup(0)->MutableDocument(0)->MutableArchiveInfo()->MutableGtaRelatedAttribute(0)->GetKey() == "gta_1");
            UNIT_ASSERT(answer.MutableGrouping(0)->MutableGroup(0)->MutableDocument(0)->MutableArchiveInfo()->MutableGtaRelatedAttribute(0)->GetValue() == "1");
        }
    }
};

UNIT_TEST_SUITE_REGISTRATION(TSearchProxyTest)

