#include "ddk_component.h"
#include "ddk_builder.h"
#include "ddk_config.h"
#include "ddk_fields.h"
#include "ddk_manager.h"
#include "ddk_parsed_entity.h"

#include <saas/library/daemon_base/daemon/messages.h>
#include <saas/library/ptr_cast/ptr_cast.h>
#include <saas/rtyserver/common/common_rty.h>
#include <saas/rtyserver/common/should_stop.h>
#include <saas/rtyserver/components/erf/erf_builder.h>
#include <saas/rtyserver/components/erf/erf_disk.h>
#include <saas/rtyserver/unistat_signals/signals.h>

#include <ysite/yandex/srchmngr/arcmgr.h>

namespace {
    const TString DDKFileName("indexddk.rty");

    template <class T>
    T& VerifiedDereference(T* p) {
        VERIFY_WITH_LOG(p, "null pointer");
        return *p;
    }
}

THolder<NRTYServer::IIndexComponentBuilder> TRTYDDKIndexComponent::CreateBuilder(const NRTYServer::TBuilderConstructionContext& context) const {
    auto base = DynamicHolderCast<TRTYErfBuilder>(TBase::CreateBuilder(context));
    VERIFY_WITH_LOG(base, "Cannot cast underlying ERF builder");
    return MakeHolder<TRTYDDKBuilder>(std::move(base), DDKConfig, Metrics, GetName());
}

THolder<NRTYServer::IIndexComponentManager> TRTYDDKIndexComponent::CreateManager(const NRTYServer::TManagerConstructionContext& context) const {
    auto base = DynamicHolderCast<IRTYErfManager>(TBase::CreateManager(context));
    VERIFY_WITH_LOG(base, "Cannot cast underlying ERF manager");
    return MakeHolder<TRTYDDKManager>(std::move(base), DDKConfig);
}

TAutoPtr<TRTYDDKManager> TRTYDDKIndexComponent::CreateManager(const TString& directory) const {
    TRTYErfDiskManager::TCreationContext cc(TPathName{directory}, DDKFileName, &GetDDKFields(), IsReadOnly);
    TAutoPtr<IRTYErfManager> erf = MakeHolder<TRTYErfDiskManager>(cc, DDK_COMPONENT_NAME).Release();
    THolder<TRTYDDKManager> ddk = THolder(MakeHolder<TRTYDDKManager>(erf, DDKConfig).Release());
    const bool valid = ddk->Open();
    return valid ? ddk.Release() : nullptr;
}

NRTYServer::IParsedEntity::TPtr TRTYDDKIndexComponent::BuildParsedEntity(NRTYServer::IParsedEntity::TConstructParams& params) const {
    return new TRTYDDKParsedEntity(params);
}

NRTYServer::IComponentParser::TPtr TRTYDDKIndexComponent::BuildParser() const {
    return new TRTYDDKComponentParser(DDKConfig);
}

TRTYDDKIndexComponent::TRTYDDKIndexComponent(const TRTYServerConfig& config)
    : TRTYErfIndexComponent(config, IsUsedStatic(config), DDKFileName, GetDDKFields())
    , IsReadOnly(config.IsReadOnly)
    , DDKConfig(VerifiedDereference(config.ComponentsConfig.Get<TDDKComponentConfig>(DDK_COMPONENT_NAME)))
{
    IndexFiles.clear();
    const bool defaultChecked = true;
    TIndexFile::EPrefetchPolicy policy = DDKConfig.GetLockDDKFile() ? TIndexFile::ppLock : TIndexFile::ppPrefetch;
    IndexFiles.insert(TIndexFile(DDKFileName, defaultChecked, policy));
    IndexFiles.insert(TIndexFile(DDKFileName + ".hdr", defaultChecked, TIndexFile::ppDisable));
    IndexFiles.insert(TIndexFile("timestamp", false, TIndexFile::ppDisable));
}

bool TRTYDDKIndexComponent::DoAllRight(const NRTYServer::TNormalizerContext& context) const {
    if (!TRTYErfIndexComponent::DoAllRight(context)) {
        ERROR_LOG << "Parent ERF component check failed" << Endl;
        return false;
    }
    auto ddk = context.Managers.GetManager<TRTYDDKManager>(DDK_COMPONENT_NAME);
    if (!ddk) {
        ERROR_LOG << "there is no ddk manager" << Endl;
        return false;
    }

    if (!CheckRemap(*ddk, context)) {
        ERROR_LOG << "Check remap failed" << Endl;
        return false;
    }

    if (!CheckHashes(*ddk, context)) {
        ERROR_LOG << "Check hashes failed" << Endl;
        return false;
    }

    NRTYServer::TIndexTimestamp timestamp(context.Dir.PathName());
    if (!timestamp.IsPresent()) {
        ERROR_LOG << "Timestamp is missing" << Endl;
        return false;
    }

    return true;
}

void TRTYDDKIndexComponent::CheckAndFix(const NRTYServer::TNormalizerContext& context) const {
    const TString& indexDirectory = context.Dir.BaseName();

    INFO_LOG << "DDK " << indexDirectory << ": Accessing indexddk" << Endl;
    TRTYErfDiskManager::TCreationContext cc(context.Dir, DDKFileName, &GetDDKFields(), IsReadOnly);
    cc.BlockCount = context.Managers.GetDocumentsCount();
    TRTYErfDiskManager erf(cc, DDK_COMPONENT_NAME);
    TRTYDDKManager ddk(&erf, DDKConfig);
    NRTYServer::TManagerGuard guard(ddk);

    if (!CheckRemap(ddk, context)) {
        NRTYServer::TInverseManagerGuard inverseGuard(ddk);
        TRTYErfIndexComponent::CheckAndFix(context);
    }

    if (!CheckHashes(ddk, context)) {
        INFO_LOG << "DDK for index " << indexDirectory << " is broken. Rebuilding." << Endl;

        TVector<TDocSearchInfo> docSearchInfos;
        auto iterator = context.Managers.GetDocSearchInfoIterator();
        AssertCorrectIndex(!!iterator, "There is no SearchDocsInfo iterator: %s", context.Dir.BaseName().data());
        for (; iterator->IsValid(); iterator->Next()) {
            if (iterator->IsDeleted())
                continue;

            docSearchInfos.push_back(iterator->Get());
        }

        TVector<ui32> toRemove;
        {
        NRTYServer::IIndexOwner::TGuardIndexModification g(context.Index);
        THashMap<TDocSearchInfo::THash, ui32> identifiers;
        TBasicFactorStorage storage(NRTYServer::NDDK::KeysCount);
        for (auto&& dsi : docSearchInfos) {
            const ui32 docid = dsi.GetDocId();
            if (!dsi.UrlInitialized())
                continue;

            erf.ReadRaw(storage, docid);
            SetHash(storage, dsi);
            erf.Write(storage, docid);

            TDocSearchInfo::THash identifier = ddk.GetIdentifier(docid);
            Y_ASSERT(identifier == dsi.GetHash());

            auto insertion = identifiers.insert(std::make_pair(identifier, docid));
            if (!insertion.second) {
                ui32 newVersion = ddk.GetVersion(docid);
                ui32 oldVersion = ddk.GetVersion(insertion.first->second);
                ERROR_LOG << "DDK " << indexDirectory << ": identifier " << identifier.Quote() << " collision "
                    << "[" << docid << " (" << newVersion << ")," << insertion.first->second << " (" << oldVersion << ")]" << Endl;
                i32 cmp = 0;
                if (newVersion == oldVersion)
                    cmp = ddk.GetTimeLiveStart(docid) > ddk.GetTimeLiveStart(insertion.first->second) ? 1 : -1;
                else
                    cmp = newVersion > oldVersion ? 1 : -1;
                ui32 victim = 0;
                if (cmp < 0) {
                    victim = docid;
                    identifiers[identifier] = insertion.first->second;
                } else {
                    victim = insertion.first->second;
                    identifiers[identifier] = docid;
                }
                toRemove.push_back(victim);
                INFO_LOG << "DDK " << indexDirectory << " docid " << victim << " removed" << Endl;
            }
        }
        }
        context.Managers.RemoveDocids(toRemove);
        INFO_LOG << "DDK " << indexDirectory << ": Rebuilt" << Endl;
    }

    NRTYServer::TIndexTimestamp timestamp(context.Dir.PathName());
    if (!timestamp.IsPresent()) {
        INFO_LOG << "DDK " << indexDirectory << ": Restoring timestamp..." << Endl;
        for (ui32 docId = 0; docId < erf.Size(); ++docId) {
            timestamp.Update(ddk.GetStreamId(docId), ddk.GetTimeLiveStart(docId));
        }
        timestamp.Flush();
        INFO_LOG << "DDK " << indexDirectory << ": Timestamp restored" << Endl;
    }
}

bool TRTYDDKIndexComponent::DoMerge(const NRTYServer::TMergeContext& ctx) const {
    TVector<NRTYServer::TManagerHolder<TRTYDDKManager>> sources;
    TVector<THolder<NRTYServer::TIndexTimestamp>> sourceTimestamps;
    TVector<THolder<NRTYServer::TIndexPositions>> sourcePositions;
    for (auto&& sourceDirectory : ctx.Context.Sources) {
        sources.push_back(CreateManager(sourceDirectory));
        sourceTimestamps.push_back(MakeHolder<NRTYServer::TIndexTimestamp>(sourceDirectory));
        sourcePositions.push_back(MakeHolder<NRTYServer::TIndexPositions>(sourceDirectory));
    }

    TVector<THolder<NRTYServer::TIndexTimestamp>> timestamps;
    TVector<THolder<NRTYServer::TIndexPositions>> positions;
    for (size_t i = 0; i < ctx.Context.Dests.size(); ++i) {
        const TString& destination = ctx.Context.Dests[i];
        timestamps.push_back(MakeHolder<NRTYServer::TIndexTimestamp>(destination));
        positions.push_back(MakeHolder<NRTYServer::TIndexPositions>(destination));
    }

    TAgeHistogramMetrics day(Metrics.DocumentAgeDay);
    TAgeHistogramMetrics hour(Metrics.DocumentAgeHour);

    ui32 oldDocCount = 0;
    day.Reset();
    hour.Reset();
    const TInstant now = Now();

    TSet<NRTYServer::TStreamId> streams;

    auto decoder = ctx.Context.Decoder;
    for (size_t segment = 0; segment < ctx.Context.Sources.size(); ++segment) {
        auto ddk = sources[segment];
        if (!ddk) {
            continue;
        }

        for (ui32 docId = 0; docId < decoder->GetSizeOfCluster(segment); ++docId) {
            auto address = decoder->Decode(segment, docId);
            if (address.DocId == REMAP_NOWHERE) {
                continue;
            }

            const ui32 stream = ddk->GetStreamId(docId);
            const TInstant timestamp = TInstant::Seconds(ddk->GetTimeLiveStart(docId));
            const TDuration age = now - timestamp;

            timestamps[address.ClusterId]->Update(stream, timestamp.Seconds(), now.Seconds());
            day.Hit(age.Days());
            hour.Hit(age.Hours());
            if (age.Days() >= GetMetricsMaxAgeDays()) {
                oldDocCount++;
            }

            if (streams.insert(stream).second) {
                INFO_LOG << "Encountered stream " << stream << " in DDK" << Endl;
            }

            if (ShouldStop(ctx.RigidStopSignal)) {
                return false;
            }
        }
    }

    for (auto&& source: sourceTimestamps) {
        for (auto stream: source->GetCurrentSnapshot()) {
            const NRTYServer::TStreamId id = stream.first;
            if (streams.contains(id)) {
                continue;
            }

            INFO_LOG << "Passing along stream " << id << " to destination indices" << Endl;
            const NRTYServer::TTs& ts = stream.second;
            if (now.Seconds() - ts.UpdateTimestamp > DDKConfig.GetMaxUpdateAge()) {
                continue;
            }
            for (auto&& destination: timestamps) {
                destination->Update(id, ts.MinValue, ts.UpdateTimestamp);
                destination->Update(id, ts.MaxValue, ts.UpdateTimestamp);
            }
        }
    }

    for (auto && destination : positions) {
        for (auto&& source : sourcePositions) {
            destination->Merge(*source);
        }
        destination->Flush();
    }

    if (!ctx.Task || !ctx.Task->GetIsPortions()) {
        Metrics.DocumentAgeDay = day;
        Metrics.DocumentAgeHour = hour;
        TSaasRTYServerSignals::UpdateOldDocCount(oldDocCount);
    }

    for (auto&& timestamp : timestamps) {
        timestamp->Flush();
    }
    sources.clear();

    return TRTYErfIndexComponent::DoMerge(ctx);
}

bool TRTYDDKIndexComponent::CheckRemap(const TRTYDDKManager& ddk, const NRTYServer::TNormalizerContext& /*context*/) const {
    if (ddk.RemapRequired()) {
        ERROR_LOG << "DDK fields changed" << Endl;
        return false;
    }
    return true;
}

bool TRTYDDKIndexComponent::CheckHashes(const TRTYDDKManager& ddk, const NRTYServer::TNormalizerContext& context) const {
    CHECK_WITH_LOG(!ddk.RemapRequired());
    auto docCount = ddk.GetDocumentsCount();
    TVector<TDocSearchInfo::THash> identifiers;
    identifiers.reserve(docCount);
    for (ui32 docId = 0; docId < docCount; ++docId) {
        if (context.Index->DocumentIsRemoved(docId))
            continue;
        identifiers.push_back(ddk.GetIdentifier(docId));
    }
    Sort(identifiers);
    for (size_t i = 1, imax = identifiers.size(); i < imax; ++i) {
        if (identifiers[i] == identifiers[i - 1]) {
            ERROR_LOG << "DDK " << context.Dir.BaseName() << ": collision detected for identifier " << identifiers[i].Quote() << Endl;
            return false;
        }
    }
    return true;
}

class TTmPruningCalcer : public TPruningConfig::ICalcer {
public:
    TTmPruningCalcer(const IDDKManager* manager)
        : Manager(manager) {
    }

    double PruningRank(ui32 docid) const override {
        VERIFY_WITH_LOG(Manager, "invlid usage");
        return static_cast<double>(Manager->GetTimeLiveStart(docid));
    }

    double PruningRankByDoc(const TPruningConfig::IDocument* document) const override {
        return static_cast<double>(document->GetTimestamp());
    }

private:
    const IDDKManager* Manager;
};

THolder<TPruningConfig::ICalcer> TRTYDDKIndexComponent::CreatePruningCalcer(const NRTYServer::IIndexManagersStorage* managers) const {
    if (Config.Pruning->GetType() == TPruningConfig::TM)
        return MakeHolder<TTmPruningCalcer>(managers ? managers->GetDDKManager() : nullptr);
    return nullptr;
}

bool TRTYDDKIndexComponent::IsUsedStatic(const TRTYServerConfig& config) {
    const TDDKComponentConfig* ddkconfig = config.ComponentsConfig.Get<TDDKComponentConfig>(DDK_COMPONENT_NAME);
    return ddkconfig == nullptr || ddkconfig->GetEnabled();
}
