#include "test_partial_run.h"

#include <kernel/doom/offroad_erf_wad/erf_io.h>
#include <kernel/doom/wad/wad.h>
#include <kernel/urlid/doc_handle.h>

#include <robot/jupiter/library/rtdoc/printer/text_utils.h>

#include <ysite/yandex/erf/flat_hash_table.h>
#include <ysite/yandex/erf_format/erf_format.h>

#include <library/cpp/offroad/flat/flat_ui64_searcher.h>
#include <library/cpp/testing/unittest/env.h>
#include <library/cpp/testing/unittest/registar.h>

#include <util/stream/str.h>

class TFusionErfModelChangeTest: public NFusion::TTestPartialRunTestBase {
public:
    UNIT_TEST_SUITE(TFusionErfModelChangeTest)
    UNIT_TEST(TestErfWithOldModel);
    UNIT_TEST_SUITE_END();
public:
    void Init() {
        BundleFilter.Exclusive = true;
        BundleFilter.DisplayNames = {
            "ReduceGroupAttrsConfigs", "PrepareGroupAttrsAndMappings", "MergeGroupAttrs",
            "MergeErf",
            "MergeInvHash"};

        IncomingLumpsFilter = {
            // for indexaa and c2n.wad
            "calculated_attrs", "content_attrs", "url_heavy_data", "group_attrs_configs",
            // for erfs
            "erf_global_lumps", "erf_doc_lumps",
            // for control
            "inv_url_hash"
        };

        TFsPath messagesDump = GetTestResource(TFsPath("erf_old_model") / "messages.protobin");
        messagesDump.CheckExists();
        MakeAllStagesInputs(messagesDump, TFsPath::Cwd() / TestWorkDir);
    }

    void SetUp() override {
        Cleanup();
        TFsPath(TestWorkDir).MkDirs();
        Init();
    }

    void TestErfWithOldModel() {
        //
        // SPI-5493 reproducer: merge some historic erf_doc_lumps
        //
        BuildAll();

        const TFsPath shard = GetFinalDir();
        const TString invHashFile = "indexinvhash.wad";
        const TString erf2wadFile = "indexerf2.wad";
        const TVector<TString> expected = ReadDocHandlesFromInvHash(shard / invHashFile);
        const TVector<TString> actual = ReadDocHandlesFromErf(shard / erf2wadFile);
        UNIT_ASSERT_VALUES_EQUAL_C(GetFinalDocCount(), actual.size(), "the number of documents in " << erf2wadFile << " dump is incorrect");

        const TFsPath testDetailsOutput = TFsPath::Cwd() / "dochandles.diff";
        const bool isSame = NRtDocTest::TTextDiffHelper::ZipDiffHelper(expected, actual, invHashFile, erf2wadFile, testDetailsOutput);
        UNIT_ASSERT_C(isSame, "erf2wad dochandles does not match invhashwad values. Details: " << testDetailsOutput);
    }

private:
    static TVector<TString> ReadDocHandlesFromInvHash(const TFsPath& file) {
        TVector<TString> lines;
        TStringStream line;
        using namespace NDoom;
        THolder<IWad> wad = IWad::Open(file);
        TBlob blob = wad->LoadGlobalLump(TWadLumpId(EWadIndexType::InvUrlHashesIndexType, EWadLumpRole::Hits));
        using TFlatSearcher = NOffroad::TFlatSearcher<ui32, ui64, NOffroad::TUi32Vectorizer, NOffroad::TUi64Vectorizer>;
        TFlatSearcher reader(blob);
        TVector<ui64> docs;
        for (ui32 index = 0; index < reader.Size(); ++index) {
            const ui32 docId = reader.ReadKey(index);
            if (docId == TFlatHashTable::EmptyDocId) {
                continue;
            }

            UNIT_ASSERT(docId < Max<ui16>());
            const ui64 data = reader.ReadData(index);
            //UNIT_ASSERT_C(data != Max<ui64>() && data != Min<ui64>() && (data >> 32) != Max<ui32>(), "InvHash has bad dochandles too");
            if (docId >= docs.size()) {
                docs.resize(docId+1, Max<ui64>());
            }
            docs[docId] = data;
        }

        for (ui32 docId = 0; docId < docs.size(); ++docId) {
            TDocHandle dh(docs[docId]);
            dh.SetUnique(true);

            line.Clear();
            line << docId << '\t' << dh;
            lines.emplace_back(std::move(line.Str()));
        }
        return lines;
    }

    static TVector<TString> ReadDocHandlesFromErf(const TFsPath& file) {
        TVector<TString> lines;
        TStringStream line;
        using namespace NDoom;
        using TIo = TErf2Io;

        TIo::TReader rd(file);

        ui32 docId;

        while (rd.ReadDoc(&docId)) {
            TIo::TReader::THit erf;
            const bool hasHit = rd.ReadHit(&erf);
            UNIT_ASSERT(hasHit);
            const ui64 data = PackUrlHash64(erf->UrlHash1, erf->UrlHash2);

            TDocHandle dh(data);
            dh.SetUnique(true);

            line.Clear();
            line << docId << '\t' << dh;
            lines.emplace_back(std::move(line.Str()));
        }
        return lines;
    }
};

UNIT_TEST_SUITE_REGISTRATION(TFusionErfModelChangeTest);


