package ru.yandex.webmaster3.storage.url.checker2;

import com.datastax.driver.core.utils.UUIDs;
import com.google.common.collect.Range;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.joda.time.DateTime;
import org.joda.time.Duration;
import org.joda.time.Instant;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import ru.yandex.webmaster3.core.WebmasterException;
import ru.yandex.webmaster3.core.data.WebmasterHostId;
import ru.yandex.webmaster3.core.http.WebmasterErrorResponse;
import ru.yandex.webmaster3.core.sitestructure.RawSearchUrlStatusEnum;
import ru.yandex.webmaster3.core.sitestructure.SearchUrlStatusEnum;
import ru.yandex.webmaster3.core.sitestructure.SearchUrlStatusUtil;
import ru.yandex.webmaster3.core.util.IdUtils;
import ru.yandex.webmaster3.core.util.TimeUtils;
import ru.yandex.webmaster3.storage.indexing2.samples.IndexingSamplesService;
import ru.yandex.webmaster3.storage.indexing2.samples.data.IndexedUrlSample;
import ru.yandex.webmaster3.storage.searchbase.SearchBaseUpdatesService;
import ru.yandex.webmaster3.storage.searchurl.SearchUrlSamplesService;
import ru.yandex.webmaster3.storage.searchurl.samples.data.SearchUrlSample;
import ru.yandex.webmaster3.storage.searchurl.samples.data.UrlStatusInfo;
import ru.yandex.webmaster3.storage.spam.ISpamHostFilter;
import ru.yandex.webmaster3.storage.url.checker2.dao.UrlCheckYtRequestsYDao;
import ru.yandex.webmaster3.storage.url.checker2.data.UrlCheckInfo;
import ru.yandex.webmaster3.storage.url.common.data.UrlCheckRequestData;
import ru.yandex.webmaster3.storage.url.common.data.UrlCheckRequestSource;
import ru.yandex.webmaster3.storage.util.ydb.exception.WebmasterYdbException;
import ru.yandex.webmaster3.storage.util.yt.*;

import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Created by leonidrom on 06/03/2017.
 */
@Component
public class UrlCheckService {
    private static final Logger log = LoggerFactory.getLogger(UrlCheckService.class);

    private static final String YT_REQUEST_TABLE_PREFIX = "urls-";
    private static final Duration YT_REQUEST_EXPIRE_PERIOD = Duration.standardHours(12);

    private static final int MAX_REQUESTS_TO_PROCESS = 1000; // чтобы не нагружать Ydb большими батчами
    private static final int MAX_REQUESTS_TABLE_SIZE = 150;

    private final YtService ytService;
    private final UrlCheckYtRequestsYDao urlCheckYtRequestsYDao;
    private final UrlCheckRequestService urlCheckRequestService;
    private final IndexingSamplesService indexingSamplesService;
    private final SearchUrlSamplesService searchUrlSamplesService;
    private final SearchBaseUpdatesService searchBaseUpdatesService;
    private final ISpamHostFilter spamHostFilter;

    private final YtPath ytRequestsPath;
    private final YtPath ytResultsPath;

    @Autowired
    public UrlCheckService(
            YtService ytService,
            UrlCheckYtRequestsYDao urlCheckYtRequestsYDao,
            UrlCheckRequestService urlCheckRequestService,
            IndexingSamplesService indexingSamplesService,
            SearchUrlSamplesService searchUrlSamplesService,
            SearchBaseUpdatesService searchBaseUpdatesService,
            ISpamHostFilter spamHostFilter,
            @Value("${external.yt.service.arnold.root.default}/checkurl/income") YtPath ytRequestsPath,
            @Value("${external.yt.service.arnold.root.default}/checkurl/outcome") YtPath ytResultsPath) {
        this.ytService = ytService;
        this.urlCheckYtRequestsYDao = urlCheckYtRequestsYDao;
        this.urlCheckRequestService = urlCheckRequestService;
        this.indexingSamplesService = indexingSamplesService;
        this.searchUrlSamplesService = searchUrlSamplesService;
        this.searchBaseUpdatesService = searchBaseUpdatesService;
        this.spamHostFilter = spamHostFilter;
        this.ytRequestsPath = ytRequestsPath;
        this.ytResultsPath = ytResultsPath;
    }

    /**
     * Возвращает данные об URL из CH, если таковые там есть, иначе формирует запрос в Yt
     */
    public Optional<UrlCheckInfo> tryGetUrlCheckInfoFromCH(UrlCheckRequestData request) {
        Optional<UrlCheckInfo> urlInfoOpt = Optional.empty();
        try {
            urlInfoOpt = getUrlCheckInfoFromCH(request);
        } catch (Throwable t) {
            // Что то пошло не так. Скорей всего, плохо CH.
            // Игнорируем, потому что ниже будем пробовать через Yt
            log.error("Failed to get URL check info from CH", t);
        }

        if (urlInfoOpt.isPresent()) {
            urlCheckRequestService.storeResult(request.getHostId(), request.getRequestId(), urlInfoOpt.get());
        } else {
            createYtRequest(request);
        }

        return urlInfoOpt;
    }

    /**
     * Возвращает данные об URL из CH, если таковые там есть
     */
    private Optional<UrlCheckInfo> getUrlCheckInfoFromCH(UrlCheckRequestData request) {
        log.debug("Getting info from CH for UUID: {}", request.getRequestId());

        URL url = request.getUrl();
        StringBuilder sb = new StringBuilder(url.getPath());
        if (url.getQuery() != null) {
            sb.append('?').append(url.getQuery());
        }
        String path = sb.toString();
        WebmasterHostId hostId = IdUtils.urlToHostId(url);

        // Страница могла быть обойдена роботом несколько раз,
        // нас интересуют результаты самого последнего обхода
        List<IndexedUrlSample> indexedSamples = indexingSamplesService.getSamples(hostId, path);
        Optional<IndexedUrlSample> indexedSampleOpt = indexedSamples.stream()
                .max(Comparator.comparing(IndexedUrlSample::getLastAccess));
        UrlCheckInfo.IndexingInfo indexingInfo = null;

        final Optional<SearchUrlSample> searchUrlSample = searchUrlSamplesService.getSearchUrlSample(hostId, path);
        if (!indexedSampleOpt.isPresent()) {
            if (searchUrlSample.isPresent()) {
                indexingInfo = new UrlCheckInfo.IndexingInfo(
                        searchUrlSample.get().getStatusInfo().getHttpCode(),
                        searchUrlSample.get().getLastAccessDateTime()
                );
            }else {
                log.debug("No indexing information found, will try Yt");
                return Optional.empty();
            }
        } else {
            IndexedUrlSample indexedSample = indexedSampleOpt.get();
            indexingInfo = new UrlCheckInfo.IndexingInfo(
                    indexedSample.getCurrentCode().getCode(),
                    indexedSample.getLastAccess()
            );
        }


        UrlCheckInfo.SearchInfo searchInfo;
        String title;
        Optional<SearchUrlSample> excludedSampleOpt = searchUrlSamplesService.getExcludedUrlSample(hostId, path);
        if (excludedSampleOpt.isPresent()) {
            SearchUrlSample excludedSample = excludedSampleOpt.get();

            // такого, чтобы исключенная из поиска страница была фейковой похоже не бывает
            boolean isFake = false;
            searchInfo = new UrlCheckInfo.SearchInfo(
                    excludedSample.getStatusInfo().getHttpCode(),
                    excludedSample.getLastAccessDateTime(),
                    excludedSample.getStatusInfo(), isFake);
            title = excludedSample.getTitle();
        } else {
            log.debug("No excluded from search information found");

            Optional<SearchUrlSample> searchSampleOpt = searchUrlSample;
            if (!searchSampleOpt.isPresent()) {
                log.debug("No search information found, will try Yt");
                return Optional.empty();
            }

            SearchUrlSample searchSample = searchSampleOpt.get();

            title = searchSample.getTitle();

            // статуса isFake для страниц в поиске в базе CH нет, поэтому проверяем так
            boolean isFake = searchSample.getLastAccessDateTime() == null;
            searchInfo = isFake ? UrlCheckInfo.SearchInfo.FAKE : new UrlCheckInfo.SearchInfo(
                    200, searchSample.getLastAccessDateTime(), searchSample.getStatusInfo(), false);
        }

        return Optional.of(new UrlCheckInfo(title, indexingInfo, searchInfo));
    }

    /**
     * Создает запрос в Yt, на случай если не нашли данные в CH
     */
    public void createYtRequest(UrlCheckRequestData request) {
        try {
            if (spamHostFilter.checkHost(request.getHostId())) {
                log.warn("Ignoring spam host {}", request.getHostId());
                return;
            }

            urlCheckYtRequestsYDao.storeRequest(request);
            log.info("Created Yt request for {}", request.getRequestId());
        } catch (WebmasterYdbException e) {
            throw new WebmasterException("Failed to create url check Yt request",
                    new WebmasterErrorResponse.YDBErrorResponse(getClass(), e), e);
        }
    }

    /**
     * Выгребает накопившиеся запросы к Yt и запускает процесс проверки URL через Yt
     *
     * @return число запросов, отгруженных в Yt
     */
    public int processYtRequests() {
        List<UrlCheckRequestData> requestsToProcess = getRequestsToProcess();
        log.debug("Found {} requests to process", requestsToProcess.size());
        if (requestsToProcess.isEmpty()) {
            return 0;
        }

        createYtRequestTable(requestsToProcess);

        try {
            urlCheckYtRequestsYDao.deleteRequests(requestsToProcess);
        } catch (WebmasterYdbException e) {
            throw new WebmasterException("Failed to store processed Yt requests",
                    new WebmasterErrorResponse.YDBErrorResponse(getClass(), e), e);
        }

        return requestsToProcess.size();
    }

    /**
     * Перекладывает результаты из Yt в наши таблички
     */
    public void processYtResults() {
        List<YtPath> resultsToProcess = getResultsToProcess();
        log.debug("Found {} result tables to process", resultsToProcess.size());
        if (resultsToProcess.isEmpty()) {
            return;
        }

        ExecutorService executorService = Executors.newFixedThreadPool(1);
        for (YtPath resultPath : resultsToProcess) {
            log.debug("Processing results from {}", resultPath);
            try {
                processYtResultTable(executorService, resultPath);
            } catch (WebmasterException e) {
                // мы не хотим, чтобы одна проблемная табличка результатов
                // навсегда блокировала обработку всех остальных табличек
                log.error("Failed to process Yt results table", e);
            }
        }

        executorService.shutdown();
        try {
            executorService.awaitTermination(5, TimeUnit.SECONDS);
        } catch (InterruptedException e) {
            log.warn("Unable to terminate executor", e);
        }
    }

    /**
     * Возвращает список запросов к Yt, которые накопились с того момента,
     * когда мы их обрабатывали в последний раз.
     */
    public List<UrlCheckRequestData> getRequestsToProcess() {
        try {
            List<UrlCheckRequestData> requestsToProcess = new ArrayList<>();
            urlCheckYtRequestsYDao.foreachRequest(requestsToProcess::add);
            if (requestsToProcess.size() > MAX_REQUESTS_TO_PROCESS) {
                requestsToProcess = new ArrayList<>(requestsToProcess.subList(0, MAX_REQUESTS_TO_PROCESS));
            }

            log.info("getRequestsToProcess: got {} requests", requestsToProcess.size());

            return requestsToProcess;
        } catch (WebmasterYdbException e) {
            throw new WebmasterException("Failed to obtain list of requests to process",
                    new WebmasterErrorResponse.YDBErrorResponse(getClass(), e), e);
        }
    }

    public static boolean isRequestExpired(UrlCheckRequestData request) {
        long expiredTS = DateTime.now().minus(YT_REQUEST_EXPIRE_PERIOD).getMillis();
        long requestTS = UUIDs.unixTimestamp(request.getRequestId());

        return (requestTS < expiredTS);
    }

    private List<YtPath> getResultsToProcess() {
        try {
            return ytService.withoutTransactionQuery(
                    cypressService -> cypressService.list(ytResultsPath));
        } catch (Exception e) {
            throw new WebmasterException("Failed to obtain list of results to process",
                    new WebmasterErrorResponse.YTServiceErrorResponse(getClass(), e));
        }
    }

    /**
     * Создает Yt табличку для запроса, что является сигналом на начало его обработки.
     * Результат обработки будет положен в другую Yt таблицу, которую мы обрабатываем
     * в другой периодической таске.
     */
    private void createYtRequestTable(List<UrlCheckRequestData> requests) {
        if (requests.isEmpty()) {
            return;
        }

        long now = System.currentTimeMillis();
        try {
            ytService.inTransaction(ytRequestsPath).execute(cypressService -> {
                long tableTs = now;
                for (int offset = 0; offset < requests.size(); offset += MAX_REQUESTS_TABLE_SIZE) {
                    String tableName = YT_REQUEST_TABLE_PREFIX + tableTs;
                    YtPath tablePath = YtPath.path(ytRequestsPath, tableName);
                    uploadToYt(cypressService, tablePath, requests.subList(offset, Math.min(offset + MAX_REQUESTS_TABLE_SIZE, requests.size())));
                    tableTs++;
                }
                return true;
            });
        } catch (YtException e) {
            throw new WebmasterException("Failed to create YT request table",
                    new WebmasterErrorResponse.YTServiceErrorResponse(getClass(), e), e);
        }
    }

    /**
     * Общение с Yt происходит путем создания таблички с двумя полями: для URL и request ID
     */
    private void uploadToYt(YtCypressService cypressService, YtPath tablePath,
                            List<UrlCheckRequestData> requests) throws YtException {

        log.trace("Table path: {}", tablePath);
        cypressService.writeTable(tablePath, tw -> {
            try {
                for (UrlCheckRequestData request : requests) {
                    tw.column(YtResultRow.F_URL, request.getUrl().toString());
                    tw.column(YtResultRow.F_REQUEST_ID, request.getRequestSource().name() + "_" + request.getRequestId().toString());
                    tw.rowEnd();
                }
            } catch (YtException e) {
                throw new RuntimeException("Unable to upload request to Yt");
            }
        });
    }

    /**
     * Вынимает результаты из Yt таблицы, кладет их в общую таблицу результатов и,
     * в случае успеха, удаляет Yt таблицу
     */
    private void processYtResultTable(ExecutorService executorService, YtPath tablePath) {
        List<YtResultRow> rows = new ArrayList<>();
        try {
            ytService.withoutTransaction(cypressService -> {
                readYtResultTable(executorService, cypressService, tablePath, rows);
                return true;
            });

            if (rows.isEmpty()) {
                throw new WebmasterException("Failed to read Yt result table (no data)",
                        new WebmasterErrorResponse.DataConsistencyErrorResponse(getClass(), "No data in Yt result table"));
            }
        } catch (YtException | InterruptedException e) {
            throw new WebmasterException("Failed to read YT results table",
                    new WebmasterErrorResponse.YTServiceErrorResponse(getClass(), e), e);
        }

        for (YtResultRow row : rows) {
            if (!row.isImportantUrl()) {
                WebmasterHostId hostId = row.getHostId();
                UUID requestId = row.getRequestId();
                log.debug("Got Yt result for UUID: {}", requestId);

                UrlCheckInfo checkInfo;
                try {
                    checkInfo = row.toUrlCheckInfo();
                } catch (Exception e) {
                    log.error("Unable to make sense of Yt result row: " + row, e);
                    throw new WebmasterException("Unable to make sense of Yt result row",
                            new WebmasterErrorResponse.DataConsistencyErrorResponse(getClass(), "Invalid row data"));
                }

                urlCheckRequestService.storeResult(hostId, requestId, checkInfo);
            }
        }

        try {
            ytService.withoutTransaction(cypressService -> {
                cypressService.remove(tablePath);
                return true;
            });
        } catch (YtException | InterruptedException e) {
            throw new WebmasterException("Failed to remove YT results table",
                    new WebmasterErrorResponse.YTServiceErrorResponse(getClass(), e), e);
        }
    }

    /**
     * Вычитывает данные из Yt таблички результатов.
     */
    private void readYtResultTable(ExecutorService executorService, YtCypressService cypressService,
                                   YtPath tablePath, List<YtResultRow> outRows) throws YtException {
        Instant currentSearchBaseDate = searchBaseUpdatesService.getSearchBaseUpdates().getCurrentBase().getBaseCollectionDate();
        AsyncTableReader<YtResultRow> tableReader =
                new AsyncTableReader<>(cypressService, tablePath, Range.all(), new YtResultRowMapper(currentSearchBaseDate),
                        YtMissingValueMode.PRINT_SENTINEL)
                        .inExecutor(executorService, "urlcheck-cacher")
                        .withRetry(5);

        log.trace("readYtResultTable: {}", tablePath);
        try (AsyncTableReader.TableIterator<YtResultRow> it = tableReader.read()) {
            while (it.hasNext()) {
                YtResultRow row = it.next();
                outRows.add(row);
            }
        } catch (IOException | InterruptedException e) {
            throw new YtException("Unable to read table: " + tablePath, e);
        }
    }

    /**
     * Строка результата из Yt
     */
    private static class YtResultRow {
        private static final Pattern REQ_ID_PATTERN = Pattern.compile("^([A-Z_]+)_([0-9a-f\\-]+)$");

        static final String F_URL = "Url";
        static final String F_REQUEST_ID = "RequestId";
        static final String F_HOST = "Host";
        static final String F_PATH = "Path";
        static final String F_ADD_TIME = "AddTime";
        static final String F_BEAUTY_URL = "BeautyUrl";
        static final String F_DESCRIPTION = "Description";
        static final String F_FOUND_BASE = "FoundBase";
        static final String F_FOUND_SPREAD = "FoundSpread";
        static final String F_HTTP_CODE = "HttpCode";
        static final String F_IS_FAKE = "IsFake";
        static final String F_IS_INDEXED = "IsIndexed";
        static final String F_IS_SEARCHABLE = "IsSearchable";
        static final String F_JUPITER_TIMESTAMP = "JupiterTimestamp";
        static final String F_LAST_ACCESS = "LastAccess";
        static final String F_MAIN_HOST = "MainHost";
        static final String F_MAIN_MIRROR_HOST = "MainMirrorHost";
        static final String F_MAIN_PATH = "MainPath";
        static final String F_MIME_TYPE = "MimeType";
        static final String F_REDIR_TARGET = "RedirTarget";
        static final String F_REL_CANONICAL_TARGET = "RelCanonicalTarget";
        static final String F_SPREAD_HTTP_CODE = "SpreadHttpCode";
        static final String F_SPREAD_LAST_ACCESS = "SpreadLastAccess";
        static final String F_SPREAD_MIME_TYPE = "SpreadMimeType";
        static final String F_TITLE = "Title";
        static final String F_URL_STATUS = "UrlStatus";

        private final Instant currentJupiterBaseDate;

        YtResultRow(Instant currentJupiterBaseDate) {
            this.currentJupiterBaseDate = currentJupiterBaseDate;
        }

        Map<String, String> fields = new HashMap<>();

        UUID getRequestId() {
            String reqIdString = fields.get(F_REQUEST_ID);
            Matcher m = REQ_ID_PATTERN.matcher(reqIdString);
            if (m.find()) {
                return UUID.fromString(m.group(2));
            } else {
                return UUID.fromString(reqIdString);
            }
        }

        WebmasterHostId getHostId() {
            return IdUtils.urlToHostId(fields.get(F_URL));
        }

        boolean hasIndexingInfo() {
            return getBoolean(F_FOUND_SPREAD);
        }

        boolean hasSearchInfo() {
            return getBoolean(F_FOUND_BASE);
        }

        boolean isSearchable() {
            return getBoolean(F_IS_SEARCHABLE);
        }

        int getIndexingHttpCode() {
            return getHttpCode(F_SPREAD_HTTP_CODE);
        }

        int getSearchHttpCode() {
            return getHttpCode(F_HTTP_CODE);
        }

        String getTitle() {
            return getString(F_TITLE);
        }

        DateTime getIndexingLastAccess() {
            return getDateTime(F_SPREAD_LAST_ACCESS);
        }

        DateTime getSearchLastAccess() {
            return getDateTime(F_LAST_ACCESS);
        }

        DateTime getAddTime() {
            return getDateTime(F_ADD_TIME);
        }

        int getUrlStatus() {
            return getInteger(F_URL_STATUS);
        }

        boolean isFake() {
            return getBoolean(F_IS_FAKE);
        }

        @Override
        public String toString() {
            return fields.toString();
        }

        boolean isImportantUrl() {
            String reqIdString = getString(F_REQUEST_ID);
            Matcher m = REQ_ID_PATTERN.matcher(reqIdString);
            return m.find() && m.group(1).equals(UrlCheckRequestSource.IMPORTANT_URL.name());
        }

        UrlCheckInfo toUrlCheckInfo() {
            String title = getTitle();

            UrlCheckInfo.IndexingInfo indexingInfo = null;
            if (hasIndexingInfo()) {
                indexingInfo = new UrlCheckInfo.IndexingInfo(
                        getIndexingHttpCode(), getIndexingLastAccess());
            }

            UrlCheckInfo.SearchInfo searchInfo = null;
            if (hasSearchInfo()) {
                if (isFake()) {
                    searchInfo = UrlCheckInfo.SearchInfo.FAKE;
                } else {
                    RawSearchUrlStatusEnum rawStatus = getRawSearchUrlStatusEnum(getUrlStatus());
                    title = SearchUrlSamplesService.hideDownloadEvidence(rawStatus, title);
                    SearchUrlStatusEnum status = SearchUrlStatusUtil.raw2View(rawStatus, isSearchable());

                    UrlStatusInfo statusInfo = new UrlStatusInfo(status, getAddTime(),
                            getString(F_BEAUTY_URL), getSearchHttpCode(), getString(F_MAIN_HOST),
                            getString(F_MAIN_PATH), getString(F_REDIR_TARGET),
                            getString(F_REL_CANONICAL_TARGET), getString(F_DESCRIPTION),
                            false, false, false, false, false);

                    searchInfo = new UrlCheckInfo.SearchInfo(
                            getSearchHttpCode(), getSearchLastAccess(), statusInfo, false);
                }
            }

            return new UrlCheckInfo(title, indexingInfo, searchInfo);
        }

        private boolean getBoolean(String field) {
            return Boolean.valueOf(fields.get(field));
        }

        private int getHttpCode(String field) {
            // в некоторых случаях, например для страниц запрещенных к индексированию
            // посредством robots.txt, http код из Yt может быть null
            String codeStr = fields.get(field);
            if (StringUtils.isNotEmpty(codeStr)) {
                return Integer.valueOf(codeStr);
            } else {
                return 0;
            }
        }

        private int getInteger(String field) {
            return Integer.valueOf(fields.get(field));
        }

        private DateTime getDateTime(String field) {
            String dateStr = fields.get(field);
            if (StringUtils.isNotEmpty(dateStr)) {
                return TimeUtils.unixTimestampToDate(Integer.valueOf(dateStr),
                        TimeUtils.EUROPE_MOSCOW_ZONE);
            } else {
                return null;
            }
        }

        private Instant getInstant(String field) {
            String dateStr = fields.get(field);
            if (StringUtils.isNotEmpty(dateStr)) {
                return new Instant(Integer.valueOf(dateStr) * 1000L);
            } else {
                return null;
            }
        }

        private String getString(String field) {
            return fields.get(field);
        }
    }

    private static class YtResultRowMapper implements YtRowMapper<YtResultRow> {
        private final Instant currentJupiterBaseDate;
        private YtResultRow row;

        YtResultRowMapper(Instant currentJupiterBaseDate) {
            this.currentJupiterBaseDate = currentJupiterBaseDate;
            this.row = new YtResultRow(currentJupiterBaseDate);
        }

        @Override
        public void nextField(String name, InputStream data) {
            try {
                String fieldStr = IOUtils.toString(data, StandardCharsets.UTF_8);
                row.fields.put(name, fieldStr);
            } catch (Exception e) {
                log.error("Unable to read Yt result field: {}", name, e);
            }
        }

        @Override
        public YtResultRow rowEnd() {
            YtResultRow r = row;
            log.trace("{}", row.fields.toString());
            row = new YtResultRow(currentJupiterBaseDate);

            return r;
        }

        @Override
        public List<String> getColumns() {
            return Arrays.asList(YtResultRow.F_HOST, YtResultRow.F_PATH, YtResultRow.F_ADD_TIME,
                    YtResultRow.F_BEAUTY_URL, YtResultRow.F_FOUND_BASE, YtResultRow.F_FOUND_SPREAD,
                    YtResultRow.F_HTTP_CODE, YtResultRow.F_IS_FAKE, YtResultRow.F_IS_INDEXED,
                    YtResultRow.F_IS_SEARCHABLE, YtResultRow.F_JUPITER_TIMESTAMP, YtResultRow.F_LAST_ACCESS,
                    YtResultRow.F_MAIN_HOST, YtResultRow.F_MAIN_MIRROR_HOST, YtResultRow.F_MAIN_PATH,
                    YtResultRow.F_MIME_TYPE, YtResultRow.F_REDIR_TARGET,
                    YtResultRow.F_REL_CANONICAL_TARGET, YtResultRow.F_DESCRIPTION, YtResultRow.F_REQUEST_ID,
                    YtResultRow.F_SPREAD_HTTP_CODE, YtResultRow.F_SPREAD_LAST_ACCESS, YtResultRow.F_SPREAD_MIME_TYPE,
                    YtResultRow.F_TITLE, YtResultRow.F_URL, YtResultRow.F_URL_STATUS);
        }
    }

    private static RawSearchUrlStatusEnum getRawSearchUrlStatusEnum(int statusInt) {
        RawSearchUrlStatusEnum status = RawSearchUrlStatusEnum.R.fromValueOrNull(statusInt);
        if (status == null) {
            status = RawSearchUrlStatusEnum.OTHER;
        }
        return status;
    }
}
