package ru.yandex.webmaster3.storage.checklist.service;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.UUID;
import java.util.function.Function;
import java.util.stream.Collectors;

import com.google.common.base.Preconditions;
import com.google.common.collect.Sets;
import lombok.RequiredArgsConstructor;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.jetbrains.annotations.NotNull;
import org.joda.time.DateTime;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import ru.yandex.webmaster3.core.checklist.data.SiteProblemState;
import ru.yandex.webmaster3.core.checklist.data.SiteProblemStorageType;
import ru.yandex.webmaster3.core.checklist.data.SiteProblemTypeEnum;
import ru.yandex.webmaster3.core.concurrency.AsyncCtx;
import ru.yandex.webmaster3.core.concurrency.AsyncTask;
import ru.yandex.webmaster3.core.data.WebmasterHostGeneration;
import ru.yandex.webmaster3.core.data.WebmasterHostId;
import ru.yandex.webmaster3.core.util.W3Collectors;
import ru.yandex.webmaster3.storage.checklist.dao.CleanableProblemsLastUpdateYDao;
import ru.yandex.webmaster3.storage.checklist.dao.CleanableProblemsYDao;
import ru.yandex.webmaster3.storage.checklist.dao.RealTimeSiteProblemsYDao;
import ru.yandex.webmaster3.storage.checklist.dao.SiteProblemsRecheckYDao;
import ru.yandex.webmaster3.storage.checklist.dao.ValidateSiteProblemService;
import ru.yandex.webmaster3.storage.checklist.data.AbstractProblemInfo;
import ru.yandex.webmaster3.storage.checklist.data.CleanableProblem;
import ru.yandex.webmaster3.storage.checklist.data.ExtendedProblem;
import ru.yandex.webmaster3.storage.checklist.data.ProblemSignal;
import ru.yandex.webmaster3.storage.checklist.data.ProblemStateInfo;
import ru.yandex.webmaster3.storage.checklist.data.RealTimeSiteProblemInfo;
import ru.yandex.webmaster3.storage.checklist.data.SiteProblemRecheckInfo;
import ru.yandex.webmaster3.storage.checklist.data.SummarySiteProblemsInfo;
import ru.yandex.webmaster3.storage.checklist.util.SiteProblemWeightUtil;
import ru.yandex.webmaster3.storage.host.service.MirrorService2;

/**
 * @author avhaliullin
 */
@RequiredArgsConstructor(onConstructor_ = @Autowired)
@Service
public class SiteProblemsService {
    private static final Logger log = LoggerFactory.getLogger(SiteProblemsService.class);
    private static final List<SiteProblemStorageType> STORAGE_TYPES = List.of(SiteProblemStorageType.values());

    private final CleanableProblemsYDao cleanableProblemsYDao;
    private final CleanableProblemsLastUpdateYDao cleanableProblemsLastUpdateYDao;
    private final SiteProblemStorageService siteProblemStorageService;
    private final MirrorService2 mirrorService2;
    private final RealTimeSiteProblemsYDao realTimeSiteProblemsYDao;
    private final SiteProblemsRecheckYDao siteProblemsRecheckYDao;
    private final SiteProblemsNotificationService siteProblemsNotificationService;
    private final List<SiteProblemsProvider> problemsProviders;
    private final ValidateSiteProblemService validateSiteProblemService;

    public WebmasterHostId toProblemHostId(WebmasterHostId hostId, SiteProblemStorageType storageType) {
        return siteProblemStorageService.toProblemHostId(hostId, storageType);
    }

    /**
     * Для подоменных проблем приводит хост к единому виду (http:domain-without-www:80)
     *
     * @param hostId
     * @return
     */
    public WebmasterHostId toProblemHostId(WebmasterHostId hostId, SiteProblemTypeEnum problemType) {
        return toProblemHostId(hostId, problemType.getStorageType());
    }

    public RealTimeSiteProblemInfo getRealTimeProblemInfo(WebmasterHostId hostId, SiteProblemTypeEnum problemType) {
        return realTimeSiteProblemsYDao.getProblemInfo(toProblemHostId(hostId, problemType), problemType);
    }

    public Map<WebmasterHostId, RealTimeSiteProblemInfo> listSitesProblems(Collection<WebmasterHostId> hostIds, SiteProblemTypeEnum problemType) {
        return realTimeSiteProblemsYDao.listSitesProblems(hostIds.stream().map(hostId -> toProblemHostId(hostId, problemType))
                .collect(Collectors.toList()), problemType);
    }

    public List<AbstractProblemInfo> listProblemsForHost(WebmasterHostId hostId, UUID hostGenerationId) {
        return listProblemsForHost(hostId, hostGenerationId, SiteProblemTypeEnum.ENABLED_PROBLEMS);
    }

    public List<AbstractProblemInfo> listProblemsForHost(WebmasterHostId hostId, UUID hostGenerationId, Set<SiteProblemTypeEnum> problemTypes) {
        return new ArrayList<>(problemsProviders.parallelStream()
                .filter(provider -> CollectionUtils.containsAny(problemTypes, provider.getSupportedProblemTypes()))
                .map(provider -> provider.listProblems(hostId, hostGenerationId))
                .flatMap(Collection::stream)
                .filter(x -> problemTypes.contains(x.getProblemType()))
                .collect(Collectors.toMap(AbstractProblemInfo::getProblemType, Function.identity(), W3Collectors.replacingMerger()))
                .values());
    }

    // Здесь гарантируется, что типы проблем в возвращаемом списке те же, что и в параметре problems
    public <T extends AbstractProblemInfo> List<ExtendedProblem<T>> getExtendedProblems(WebmasterHostId hostId, List<T> problems) {
        Map<WebmasterHostId, Set<SiteProblemStorageType>> typeByHost =
                STORAGE_TYPES.stream().collect(Collectors.toMap(type -> toProblemHostId(hostId, type), Set::of, Sets::union));

        Map<SiteProblemTypeEnum, SiteProblemRecheckInfo> recheckMap = typeByHost.entrySet().parallelStream()
                .flatMap(entry ->
                        siteProblemsRecheckYDao.getRecheckInfoForHost(entry.getKey()).stream()
                                .filter(problemHostId -> entry.getValue().contains(problemHostId.getProblemType().getStorageType())))
                .collect(Collectors.toMap(SiteProblemRecheckInfo::getProblemType, x -> x, (a, b) -> a));

        return problems.stream()
                .map(p -> {
                    boolean recheckInProgress = false;
                    boolean recheckFailed = false;
                    DateTime recheckRequestDate = null;

                    SiteProblemRecheckInfo recheckInfo = recheckMap.get(p.getProblemType());
                    if (recheckInfo != null && p.getProblemType().isRecheckable(p.getState(), p.getContent())) {
                        SiteProblemRecheckInfo.RecheckState recheckState = recheckInfo.getState(p.getLastUpdate());
                        recheckRequestDate = recheckInfo.getRequestDate();
                        recheckInProgress = recheckState.isInProgres;
                        recheckFailed = recheckState.isFailed;
                    }

                    return new ExtendedProblem<>(p, SiteProblemWeightUtil.computeWeight(p.getProblemType()),
                            recheckInProgress, recheckRequestDate, recheckFailed);
                })
                .collect(Collectors.toList());
    }

    public void notifyCleanableProblemUpdateFinished(SiteProblemTypeEnum problemType, DateTime updateStartTime) {
        cleanableProblemsLastUpdateYDao.updateProblemType(problemType, updateStartTime);
    }

    public void touchCleanableProblemIfExists(WebmasterHostId hostId, SiteProblemTypeEnum problemType) {
        if (!validateSiteProblemService.hostInWebmaster(hostId, problemType)) {
            return;
        }

        var problemHostId = toProblemHostId(hostId, problemType);
        CleanableProblem problem = cleanableProblemsYDao.getProblemInfo(problemHostId, problemType);
        if (problem != null) {
            cleanableProblemsYDao.updateProblem(new CleanableProblem(problemType, DateTime.now(), problemHostId));
        }
    }

    public void updateCleanableProblem(@NotNull WebmasterHostId hostId,
                                       @NotNull ProblemSignal problem,
                                       DateTime actualSinceForNewProblem) {
        if (!validateSiteProblemService.hostInWebmaster(hostId, problem.getProblemType())) {
            return;
        }

        var problemHostId = toProblemHostId(hostId, problem.getProblemType());
        if (updateRealTimeProblem(problemHostId, problem, false, false, actualSinceForNewProblem)) {
            if (problem.getState() == problem.getProblemType().getDefaultState()) {
                cleanableProblemsYDao.deleteProblem(problemHostId, problem.getProblemType());
            } else {
                cleanableProblemsYDao.updateProblem(new CleanableProblem(problem.getProblemType(), DateTime.now(), problemHostId));
            }
        }
    }

    public void updateCleanableProblem(@NotNull WebmasterHostId hostId,
                                       @NotNull ProblemSignal problem) {
        updateCleanableProblem(hostId, problem, DateTime.now());
    }

    public void updateCleanableProblems(Map<WebmasterHostId, ProblemSignal> problemMap, SiteProblemTypeEnum problemType) {
        problemMap = problemMap.entrySet().stream()
                .filter(e -> validateSiteProblemService.hostInWebmaster(e.getKey(), problemType))
                .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
        if (problemMap.isEmpty()) {
            return;
        }

        Set<WebmasterHostId> problemHostIds = problemMap.keySet().stream()
                .map(hostId -> toProblemHostId(hostId, problemType))
                .collect(Collectors.toSet());
        Map<WebmasterHostId, RealTimeSiteProblemInfo> curProblemInfos =
                realTimeSiteProblemsYDao.listSitesProblems(problemHostIds, problemType);

        Set<CleanableProblem> newCleanableProblems = new HashSet<>();
        Set<WebmasterHostId> hostsWithoutProblem = new HashSet<>();
        List<RealTimeSiteProblemInfo> realTimeSiteProblemInfos = new ArrayList<>();
        Map<WebmasterHostId, Pair<RealTimeSiteProblemInfo, ProblemSignal>> eventSend = new HashMap<>();
        problemMap.forEach((hostId, problemSignal) -> {
            var problemHostId = toProblemHostId(hostId, problemSignal.getProblemType());
            var info = batchUpdateRealTimeProblem(problemHostId, problemSignal, false, curProblemInfos.get(problemHostId));

            if (info != null) {
                final RealTimeSiteProblemInfo problemInfo = info.getKey();
                realTimeSiteProblemInfos.add(problemInfo);
                if (info.getValue()) {
                    eventSend.put(problemInfo.getHostId(), Pair.of(problemInfo, problemSignal));
                }
                if (problemInfo.getState() == problemInfo.getProblemType().getDefaultState()) {
                    hostsWithoutProblem.add(problemHostId);
                } else {
                    newCleanableProblems.add(new CleanableProblem(problemType, DateTime.now(), problemHostId));
                }
            }
        });
        realTimeSiteProblemsYDao.addProblems(realTimeSiteProblemInfos);
        siteProblemsNotificationService.sendNotification(eventSend);
        cleanableProblemsYDao.updateProblems(newCleanableProblems);
        cleanableProblemsYDao.deleteProblems(hostsWithoutProblem, problemType);
    }

    public boolean updateRealTimeProblem(@NotNull WebmasterHostId hostId, @NotNull ProblemSignal problem) {
        return updateRealTimeProblem(hostId, problem, false, false);
    }

    public void updateRealTimeProblem(Map<WebmasterHostId, Pair<ProblemSignal, RealTimeSiteProblemInfo>> map) {
        List<RealTimeSiteProblemInfo> realTimeSiteProblemInfos = new ArrayList<>();
        Map<WebmasterHostId, Pair<RealTimeSiteProblemInfo, ProblemSignal>> eventSend = new HashMap<>();
        for (Map.Entry<WebmasterHostId, Pair<ProblemSignal, RealTimeSiteProblemInfo>> entry : map.entrySet()) {
            WebmasterHostId hostId = entry.getKey();
            ProblemSignal problem = entry.getValue().getKey();
            RealTimeSiteProblemInfo curInfo = entry.getValue().getValue();
            if (!validateSiteProblemService.hostInWebmaster(hostId, problem.getProblemType())) {
                continue;
            }

            var problemHostId = toProblemHostId(hostId, problem.getProblemType());
            final Pair<RealTimeSiteProblemInfo, Boolean> problemInfo = batchUpdateRealTimeProblem(problemHostId, problem, false, curInfo);
            if (problemInfo != null) {
                realTimeSiteProblemInfos.add(problemInfo.getKey());
                if (problemInfo.getValue()) {
                    eventSend.put(problemInfo.getKey().getHostId(), Pair.of(problemInfo.getKey(), problem));
                }
            }
        }
        realTimeSiteProblemsYDao.addProblems(realTimeSiteProblemInfos);
        siteProblemsNotificationService.sendNotification(eventSend);
    }

    public boolean updateRealTimeProblem(@NotNull WebmasterHostId hostId, @NotNull ProblemSignal problem,
                                         boolean forceDefaultState, boolean withoutCheck) {
        return updateRealTimeProblem(hostId, problem, forceDefaultState, withoutCheck, DateTime.now());
    }

    public boolean updateRealTimeProblem(@NotNull WebmasterHostId hostId, @NotNull ProblemSignal problem,
                                         boolean forceDefaultState, boolean withoutCheck, DateTime actualSinceForNewProblem) {
        if (!withoutCheck && !validateSiteProblemService.hostInWebmaster(hostId, problem.getProblemType())) {
            return false;
        }

        var problemHostId = toProblemHostId(hostId, problem.getProblemType());
        RealTimeSiteProblemInfo curInfo = realTimeSiteProblemsYDao.getProblemInfo(problemHostId, problem.getProblemType());
        return updateRealTimeProblem(problemHostId, problem, forceDefaultState, curInfo, actualSinceForNewProblem);
    }

    public void updateRealTimeSiteProblems(Map<WebmasterHostId, List<ProblemSignal>> map, SiteProblemStorageType storageType) {
        List<Pair<RealTimeSiteProblemInfo, ProblemSignal>> list = new ArrayList<>();
        Map<WebmasterHostId, Pair<RealTimeSiteProblemInfo, ProblemSignal>> eventSend = new HashMap<>();
        final List<Pair<WebmasterHostId, WebmasterHostId>> hostIds = map
                .entrySet()
                .stream()
                .filter(e -> !e.getValue().isEmpty() && validateSiteProblemService.hostInWebmaster(e.getKey(), storageType))
                .map(e -> e.getKey())
                .map(e -> Pair.of(e, toProblemHostId(e, storageType)))
                .collect(Collectors.toList());
        final Map<WebmasterHostId, List<RealTimeSiteProblemInfo>> webmasterHostIdListMap
                = realTimeSiteProblemsYDao.listSitesProblems(hostIds.stream().map(Pair::getValue).collect(Collectors.toList()));
        for (Pair<WebmasterHostId, WebmasterHostId> pairHostId : hostIds) {
            WebmasterHostId hostId = pairHostId.getKey();
            List<ProblemSignal> problems = map.get(hostId);
            var problemHostId = pairHostId.getValue();
            List<RealTimeSiteProblemInfo> currentProblems = webmasterHostIdListMap.get(problemHostId);
            if (currentProblems == null) {
                continue;
            }
            Map<SiteProblemTypeEnum, RealTimeSiteProblemInfo> problemMap = currentProblems.stream()
                    .collect(W3Collectors.toEnumMap(RealTimeSiteProblemInfo::getProblemType, Function.identity(), SiteProblemTypeEnum.class));
            for (ProblemSignal problem : problems) {
                final Pair<RealTimeSiteProblemInfo, Boolean> problemInfo = batchUpdateRealTimeProblem(problemHostId, problem, false, problemMap.get(problem.getProblemType()));
                if (problemInfo != null) {
                    list.add(Pair.of(problemInfo.getKey(), problem));
                    if (problemInfo.getValue()) {
                        eventSend.put(problemInfo.getKey().getHostId(), Pair.of(problemInfo.getKey(), problem));
                    }
                }
            }
        }
        if (!list.isEmpty()) {
            realTimeSiteProblemsYDao.addProblems(list.stream().map(Pair::getLeft).collect(Collectors.toList()));
            siteProblemsNotificationService.sendNotification(eventSend);
        }
    }

    public void updateRealTimeSiteProblems(Map<WebmasterHostId, List<ProblemSignal>> map) {
        List<Pair<RealTimeSiteProblemInfo, ProblemSignal>> list = new ArrayList<>();
        Map<WebmasterHostId, Pair<RealTimeSiteProblemInfo, ProblemSignal>> eventSend = new HashMap<>();
        for (Map.Entry<WebmasterHostId, List<ProblemSignal>> entry : map.entrySet()) {
            WebmasterHostId hostId = entry.getKey();
            List<ProblemSignal> problems = entry.getValue().stream().
                    filter(e -> validateSiteProblemService.hostInWebmaster(hostId, e.getProblemType().getStorageType()))
                    .collect(Collectors.toList());
            if (problems.isEmpty()) {
                continue;
            }
            final Set<SiteProblemStorageType> collect =
                    problems.stream().map(e -> e.getProblemType().getStorageType()).collect(Collectors.toSet());
            for (SiteProblemStorageType storageType : collect) {
                var problemHostId = toProblemHostId(hostId, storageType);

                List<RealTimeSiteProblemInfo> currentProblems = realTimeSiteProblemsYDao.listSiteProblems(problemHostId);
                Map<SiteProblemTypeEnum, RealTimeSiteProblemInfo> problemMap = currentProblems.stream()
                        .collect(W3Collectors.toEnumMap(RealTimeSiteProblemInfo::getProblemType, Function.identity(), SiteProblemTypeEnum.class));
                for (ProblemSignal problem : problems) {
                    final Pair<RealTimeSiteProblemInfo, Boolean> problemInfo = batchUpdateRealTimeProblem(problemHostId, problem, false, problemMap.get(problem.getProblemType()));

                    if (problemInfo != null) {
                        list.add(Pair.of(problemInfo.getKey(), problem));
                        if (problemInfo.getValue()) {
                            eventSend.put(problemInfo.getKey().getHostId(), Pair.of(problemInfo.getKey(), problem));
                        }
                    }
                }
            }
        }
        if (!list.isEmpty()) {
            realTimeSiteProblemsYDao.addProblems(list.stream().map(Pair::getLeft).collect(Collectors.toList()));
            siteProblemsNotificationService.sendNotification(eventSend);
        }
    }


    private Pair<RealTimeSiteProblemInfo, Boolean> batchUpdateRealTimeProblem(@NotNull WebmasterHostId hostId, @NotNull ProblemSignal problem,
                                                                              boolean forceSaveDefaultState, RealTimeSiteProblemInfo curInfo) {
        SiteProblemTypeEnum problemType = problem.getProblemType();
        SiteProblemState problemState = problem.getState();
        SiteProblemStorageType storageType = problemType.getStorageType();
        Preconditions.checkArgument(storageType.isRealTimeProblem(), "Bad problem: " + problem);
        Preconditions.checkArgument(hostId.equals(toProblemHostId(hostId, storageType)), "Bad hostId " + hostId + " in problem signal");
        RealTimeSiteProblemInfo answer = null;
        // если перед нами неглавное зеркало и неподходящая проблема - сбросим статус
        if (problemState == SiteProblemState.PRESENT && problemType.isOnlyForMainMirror() &&
                !mirrorService2.isMainMirror(hostId)) {
            log.info("Skipping problem {} for non-main mirror host {}", problemType, hostId);
            problem = new ProblemSignal(problemType, SiteProblemState.ABSENT, problem.getLastUpdate());
            problemState = SiteProblemState.ABSENT;
        }
        DateTime now = DateTime.now();
        int weight = SiteProblemWeightUtil.computeWeight(problemType);
        if (curInfo == null) {
            log.info("Updating realtime problem {} for host {}. " +
                            "New: state {}, update date {}. " +
                            "Prev info is null",
                    problemType, hostId,
                    problemState, problem.getLastUpdate()
            );
        } else {
            log.info("Updating realtime problem {} for host {}. " +
                            "New: state {}, update date {}. " +
                            "Prev: state {}, update date {}, last flushed {}, actual since {}",
                    problemType, hostId,
                    problemState, problem.getLastUpdate(),
                    curInfo.getState(), curInfo.getLastUpdate(), curInfo.getLastFlushed(), curInfo.getActualSince()
            );
        }
        boolean sendEvent = false;
        if (curInfo == null) {
            if (forceSaveDefaultState || problemState != problemType.getDefaultState()) {
                DateTime actualSince = problemState.isPresent() ? now : null;
                log.info("Adding problem {} for host {}", problemType, hostId);
                answer = new RealTimeSiteProblemInfo(hostId, problem.getLastUpdate(),
                        actualSince, null, problemState, problemType, problem.getContent(), weight);
                sendEvent = true;
            }
        } else {
            if (curInfo.getLastUpdate() == null || !curInfo.getLastUpdate().isAfter(problem.getLastUpdate())) {
                if (curInfo.getState() == problemState) {
                    log.info("No state change, will touch problem {} for host {}", problemType, hostId);
                    if (curInfo.getState() == SiteProblemState.PRESENT) {
                        answer = curInfo.withContent(problem.getContent(), problem.getLastUpdate());
                    } else {
                        answer = curInfo.withLastUpdate(problem.getLastUpdate());
                    }
                } else {
                    boolean changedPresentState = curInfo.getState().isPresent() != problemState.isPresent();

                    boolean actualSinceToday = curInfo.getActualSince() == null ||
                            !curInfo.getActualSince().withTimeAtStartOfDay().isBefore(now.withTimeAtStartOfDay());

                    DateTime lastFlushed;
                    DateTime actualSince;

                    if (changedPresentState) {
                        if (actualSinceToday) {
                            actualSince = curInfo.getLastFlushed();
                            lastFlushed = curInfo.getLastFlushed();
                        } else {
                            actualSince = now;
                            lastFlushed = curInfo.getActualSince();
                        }
                    } else {
                        lastFlushed = curInfo.getLastFlushed();
                        actualSince = curInfo.getActualSince();
                    }

                    if (problemState == SiteProblemState.PRESENT) {
                        log.info("Adding problem {} for host {}", problemType, hostId);
                        answer = new RealTimeSiteProblemInfo(hostId, problem.getLastUpdate(),
                                actualSince, lastFlushed, problemState, problemType,
                                problem.getContent(), weight);
                    } else {
                        log.info("Fixed problem {} for host {}", problemType, hostId);
                        answer = curInfo.withNewState(
                                problemState, actualSince, lastFlushed, problem.getLastUpdate());
                    }
                    sendEvent = true;
                }
            } else {
                log.info("Ignoring new problem {} state for host {}: new lastUpdate {} is before current {}",
                        problemType, hostId, problem.getLastUpdate(), curInfo.getLastUpdate());
                return null;
            }
        }
        return answer == null ? null : Pair.of(answer, sendEvent);
    }

    private boolean updateRealTimeProblem(@NotNull WebmasterHostId hostId, @NotNull ProblemSignal problem,
                                          boolean forceSaveDefaultState, RealTimeSiteProblemInfo curInfo,
                                          DateTime actualSinceForNewProblem) {
        SiteProblemTypeEnum problemType = problem.getProblemType();
        SiteProblemState problemState = problem.getState();
        SiteProblemStorageType storageType = problemType.getStorageType();
        Preconditions.checkArgument(storageType.isRealTimeProblem(), "Bad problem: " + problem);
        Preconditions.checkArgument(hostId.equals(toProblemHostId(hostId, storageType)), "Bad hostId " + hostId + " in problem signal");

        // если перед нами неглавное зеркало и неподходящая проблема - сбросим статус
        if (problemState == SiteProblemState.PRESENT && problemType.isOnlyForMainMirror() &&
                !mirrorService2.isMainMirror(hostId)) {
            log.info("Skipping problem {} for non-main mirror host {}", problemType, hostId);
            problem = new ProblemSignal(problemType, SiteProblemState.ABSENT, problem.getLastUpdate());
            problemState = SiteProblemState.ABSENT;
        }
        DateTime now = DateTime.now();
        int weight = SiteProblemWeightUtil.computeWeight(problemType);
        if (curInfo == null) {
            log.info("Updating realtime problem {} for host {}. " +
                            "New: state {}, update date {}. " +
                            "Prev info is null",
                    problemType, hostId,
                    problemState, problem.getLastUpdate()
            );
        } else {
            log.info("Updating realtime problem {} for host {}. " +
                            "New: state {}, update date {}. " +
                            "Prev: state {}, update date {}, last flushed {}, actual since {}",
                    problemType, hostId,
                    problemState, problem.getLastUpdate(),
                    curInfo.getState(), curInfo.getLastUpdate(), curInfo.getLastFlushed(), curInfo.getActualSince()
            );
        }
        if (curInfo == null) {
            if (forceSaveDefaultState || problemState != problemType.getDefaultState()) {
                DateTime actualSince = problemState.isPresent() ? actualSinceForNewProblem : null;
                log.info("Adding problem {} for host {}", problemType, hostId);
                realTimeSiteProblemsYDao.addProblem(new RealTimeSiteProblemInfo(hostId, problem.getLastUpdate(),
                        actualSince, null, problemState, problemType, problem.getContent(), weight));
                siteProblemsNotificationService.sendNotification(hostId, problem, curInfo, actualSince);
            }
        } else {
            if (curInfo.getLastUpdate() == null || !curInfo.getLastUpdate().isAfter(problem.getLastUpdate())) {
                if (curInfo.getState() == problemState) {
                    log.info("No state change, will touch problem {} for host {}", problemType, hostId);
                    if (curInfo.getState() == SiteProblemState.PRESENT) {
                        realTimeSiteProblemsYDao.addProblem(curInfo.withContent(problem.getContent(), problem.getLastUpdate()));
                    } else {
                        realTimeSiteProblemsYDao.addProblem(curInfo.withLastUpdate(problem.getLastUpdate()));
                    }
                } else {
                    boolean changedPresentState = curInfo.getState().isPresent() != problemState.isPresent();

                    boolean actualSinceToday = curInfo.getActualSince() == null ||
                            !curInfo.getActualSince().withTimeAtStartOfDay().isBefore(now.withTimeAtStartOfDay());

                    DateTime lastFlushed;
                    DateTime actualSince;

                    if (changedPresentState) {
                        if (actualSinceToday) {
                            actualSince = curInfo.getLastFlushed();
                            lastFlushed = curInfo.getLastFlushed();
                        } else {
                            actualSince = now;
                            lastFlushed = curInfo.getActualSince();
                        }
                    } else {
                        lastFlushed = curInfo.getLastFlushed();
                        actualSince = curInfo.getActualSince();
                    }

                    if (problemState == SiteProblemState.PRESENT) {
                        log.info("Adding problem {} for host {}", problemType, hostId);
                        realTimeSiteProblemsYDao.addProblem(new RealTimeSiteProblemInfo(hostId, problem.getLastUpdate(),
                                actualSince, lastFlushed, problemState, problemType,
                                problem.getContent(), weight));
                    } else {
                        log.info("Fixed problem {} for host {}", problemType, hostId);
                        realTimeSiteProblemsYDao.addProblem(curInfo.withNewState(
                                problemState, actualSince, lastFlushed, problem.getLastUpdate()));
                    }
                    siteProblemsNotificationService.sendNotification(hostId, problem, curInfo, actualSince);
                }
            } else {
                log.info("Ignoring new problem {} state for host {}: new lastUpdate {} is before current {}",
                        problemType, hostId, problem.getLastUpdate(), curInfo.getLastUpdate());
                return false;
            }
        }
        return true;
    }

    public SummarySiteProblemsInfo getRealTimeProblemsSummary(WebmasterHostId hostId) {
        return summarizeProblems(hostId, listProblemsForHost(hostId, null, SiteProblemTypeEnum.NON_GENERATION_BASED_PROBLEMS));
    }

    public Map<WebmasterHostId, SummarySiteProblemsInfo> getRealTimeProblemsSummary(AsyncCtx ctx, Collection<WebmasterHostId> hostIds) {
        if (hostIds.isEmpty()) {
            return Collections.emptyMap();
        }
        List<WebmasterHostGeneration> hostGenerations = hostIds.stream().map(WebmasterHostGeneration::createEmpty).collect(Collectors.toList());
        List<AsyncTask<Map<WebmasterHostId, List<? extends AbstractProblemInfo>>>> problemsTasks = problemsProviders.stream()
                .filter(provider -> CollectionUtils.containsAny(SiteProblemTypeEnum.NON_GENERATION_BASED_PROBLEMS, provider.getSupportedProblemTypes()))
                .map(provider -> ctx.fork(() -> provider.listProblems(hostGenerations)))
                .collect(Collectors.toList());

        return problemsTasks.stream()
                .map(AsyncTask::join)
                .map(Map::values)
                .flatMap(Collection::stream)
                .flatMap(Collection::stream)
                .collect(Collectors.groupingBy(AbstractProblemInfo::getHostId))
                .entrySet().stream()
                .map(entry -> Pair.of(entry.getKey(), summarizeProblems(entry.getKey(), entry.getValue())))
                .collect(W3Collectors.toHashMap());
    }


    private SummarySiteProblemsInfo summarizeProblems(WebmasterHostId hostId, Collection<? extends ProblemStateInfo> problems) {
        boolean isMainMirror = mirrorService2.isMainMirror(hostId);
        Map<SiteProblemTypeEnum, SiteProblemState> problem2State = problems
                .stream()
                .filter(p -> !p.getProblemType().isDisabled())
                .filter(p -> p.getState() == SiteProblemState.PRESENT)
                .filter(p -> isMainMirror || p.getProblemType().isApplicableForNonMainMirror())
                .collect(Collectors.toMap(ProblemStateInfo::getProblemType, ProblemStateInfo::getState, W3Collectors.replacingMerger()));
        return new SummarySiteProblemsInfo(0, 0, problem2State);
    }
}
