use hahn;
--pragma yt.AutoMerge="disabled";
--pragma yt.DefaultMemoryLimit="8G";
pragma yt.MaxRowWeight = '128M';

$ROOT                   = "//home/webmaster/prod/analytics/checklist/";

$input_hosts_verified   = "//home/webmaster/prod/analytics/mailings/hosts-verified";
$input_webmaster_visits = "//home/webmaster/prod/analytics/logs/webmaster-visits";
$input_site_problems    = $ROOT || "site-problems-history";
$output_visits          = $ROOT || "statistics/visits";
$output_source          = $ROOT || "statistics/source";
$output_history         = $ROOT || "statistics/history";
$output_extracted       = $ROOT || "statistics/extracted";
$output_stats           = $ROOT || "statistics/stats";
$output_report_tail     = $ROOT || "statistics/report-tail";
$output_report          = $ROOT || "statistics/report";

$script = @@
import cyson

def f(history):
    had_problem = False
    history_sorted = sorted(history, key=lambda x: x[b"Period"])[-90:]

    problem_periods = 0
    total_periods   = len(history_sorted)
    visits                      = 0
    visits_diag                 = 0
    visits_during_problem       = 0
    visits_diag_during_problem  = 0

    for rec in history_sorted:
        if rec[b"IsThereVisit"]:
            visits += 1
        if rec[b"IsThereVisitDiagnostics"]:
            visits_diag += 1

        if rec[b"IsProblemExists"]:
            if rec[b"IsThereVisit"]:
                visits_during_problem += 1
            if rec[b"IsThereVisitDiagnostics"]:
                visits_diag_during_problem += 1
            had_problem = True
            problem_periods += 1

    last_problem = False
    if len(history_sorted) > 0:
        last_problem = history_sorted[-1][b"IsProblemExists"]

    return {
        "HadProblem": had_problem,
        "LastProblem": last_problem,
        "ProblemPeriods": problem_periods,
        "TotalPeriods": total_periods,
        "Visits": visits,
        "VisitsDuringProblem": visits_during_problem,
        "VisitsDiagnostics": visits_diag,
        "VisitsDiagnosticsDuringProblem": visits_diag_during_problem,
    }

f._yql_convert_yson = (cyson.loads, cyson.dumps)
@@;

$udf = Python3::f(Callable<(Yson?)->Yson?>, $script);

$is_there_problem = ($problems, $problem_type) -> {
    return ListLength(
        ListFilter(
            Yson::ConvertToList($problems),
            ($problem) -> {
                return Yson::ConvertToInt64($problem) == $problem_type;
            }
        )
    ) > 0
};

$is_there_visit = ($visits) -> {
    return ListLength(
        Yson::ConvertToList($visits)
    ) > 0
};

$is_there_visit_diagnostics = ($visits) -> {
    return ListHas(
        Yson::ConvertToStringList($visits),
        "/checklist/info"
    );
};

$extract_problem = ($history, $problem_type) -> {
    return Yson::Serialize(Yson::From(ListSort(
        ListMap(
            Yson::ConvertToList($history),
            ($record) -> {
                return AsStruct(
                    $record.Period as Period,
                    $is_there_problem($record.SiteProblems, $problem_type) as IsProblemExists,
                    $is_there_visit($record.Visits) as IsThereVisit,
                    $is_there_visit_diagnostics($record.Visits) as IsThereVisitDiagnostics
                );
            }
        ),
        ($x) -> {
            return Yson::ConvertToString($x.Period);
        }
    )));
};

$had_problem = ($record) -> {
    return Yson::ConvertToBool($record.HadProblem) == true
};

$is_problem_fixed = ($record) -> {
    return Yson::ConvertToBool($record.HadProblem) == true
        and Yson::ConvertToBool($record.LastProblem) == false
};

$visit_during_problem = ($record) -> {
    return Yson::ConvertToInt64($record.VisitsDuringProblem) > 0
};

$visit_diag_during_problem = ($record) -> {
    return Yson::ConvertToInt64($record.VisitsDiagnosticsDuringProblem) > 0
};

$get_problem_periods = ($record) -> {
    return Yson::ConvertToInt64($record.ProblemPeriods);
};

$get_stat = ($record) -> {
    return AsStruct(
        $had_problem($record) as HadProblem,
        $is_problem_fixed($record) as Fixed,
        ($is_problem_fixed($record) and $visit_during_problem($record)) as FixedAfterVisit,
        ($is_problem_fixed($record) and $visit_diag_during_problem($record)) as FixedAfterVisitDiag,
        $get_problem_periods($record) as ProblemFixedIn,
    );
};

$agg_p90 = AggregationFactory("percentile", 0.90);
$agg_p75 = AggregationFactory("percentile", 0.75);

$hosts = (
    select distinct Host
    from (
        select Host
        from $input_site_problems
        union all
        select HostUrl as Host
        from $input_hosts_verified 
    )
);

$periods = (
    select distinct Period
    from $input_site_problems
);

$report_date = (
    select max(Period)
    from $periods
);

$hosts_periods = (
    select Host, Period
    from $hosts as h
    cross join $periods as p
);

$visits = (
    select 
        Host,
        Period,
        aggregate_list_distinct(Route) as Visits
    from $input_webmaster_visits
    where Host is not null
        and Route is not null
    group by
        Host,
        Period
);

$source = (
    select 
        hp.Host as Host,
        hp.Period as Period,
        coalesce(sp.SiteProblems, AsList()) as SiteProblems,
        coalesce(v.Visits, AsList()) as Visits,
        $report_date as ReportDate,
    from $hosts_periods as hp
    left join $input_site_problems as sp
        on hp.Host == sp.Host
        and hp.Period == sp.Period
    left join $visits as v
        on hp.Host == v.Host
        and hp.Period == v.Period
);

$history = (
    select 
        Host,
        ReportDate,
        Yson::Serialize(Yson::From(
            ListSort(
                ListFilter(
                    aggregate_list(
                        AsStruct(
                            Period as Period,
                            SiteProblems as SiteProblems,
                            Visits as Visits
                        )
                    ),
                    ($x) -> { return $x.SiteProblems is not null; }
                ),
                ($x) -> { return $x.Period; }
            )
        )) as History,
    from $source
    group by Host, ReportDate
);

$extracted = (
    select
        Host,
        ReportDate,
        $extract_problem(History, 9) as DISALLOWED_IN_ROBOTS,
        $extract_problem(History, 10) as DNS_ERROR,
        $extract_problem(History, 41) as SSL_CERTIFICATE_ERROR,
        $extract_problem(History, 25) as TOO_MANY_BROKEN_LINKS,
        $extract_problem(History, 49) as NO_METRIKA_COUNTER_BINDING,
        $extract_problem(History, 44) as FAVICON_ERROR,
        $extract_problem(History, 4) as SITEMAP_NOT_SET,
        $extract_problem(History, 46) as NO_METRIKA_COUNTER_CRAWL_ENABLED,
        $extract_problem(History, 5) as ERRORS_IN_SITEMAPS,
        $extract_problem(History, 21) as DOCUMENTS_MISSING_TITLE,
        $extract_problem(History, 22) as DOCUMENTS_MISSING_DESCRIPTION,
        $extract_problem(History, 12) as ROBOTS_TXT_ERROR,
        $extract_problem(History, 18) as NO_ROBOTS_TXT,
        $extract_problem(History, 11) as NO_404_ERRORS,
        $extract_problem(History, 16) as MISSING_FAVICON,
        $extract_problem(History, 26) as NOT_MOBILE_FRIENDLY,
        $extract_problem(History, 51) as BIG_FAVICON_ABSENT,
        $extract_problem(History, 14) as MORDA_ERROR,
        $extract_problem(History, 52) as CONNECT_FAILED,
    from $history
);

$stats = (
    select 
        Host,
        ReportDate,
        $udf(DISALLOWED_IN_ROBOTS) as DISALLOWED_IN_ROBOTS,
        $udf(DNS_ERROR) as DNS_ERROR,
        $udf(SSL_CERTIFICATE_ERROR) as SSL_CERTIFICATE_ERROR,
        $udf(TOO_MANY_BROKEN_LINKS) as TOO_MANY_BROKEN_LINKS,
        $udf(NO_METRIKA_COUNTER_BINDING) as NO_METRIKA_COUNTER_BINDING,
        $udf(FAVICON_ERROR) as FAVICON_ERROR,
        $udf(SITEMAP_NOT_SET) as SITEMAP_NOT_SET,
        $udf(NO_METRIKA_COUNTER_CRAWL_ENABLED) as NO_METRIKA_COUNTER_CRAWL_ENABLED,
        $udf(ERRORS_IN_SITEMAPS) as ERRORS_IN_SITEMAPS,
        $udf(DOCUMENTS_MISSING_TITLE) as DOCUMENTS_MISSING_TITLE,
        $udf(DOCUMENTS_MISSING_DESCRIPTION) as DOCUMENTS_MISSING_DESCRIPTION,
        $udf(ROBOTS_TXT_ERROR) as ROBOTS_TXT_ERROR,
        $udf(NO_ROBOTS_TXT) as NO_ROBOTS_TXT,
        $udf(NO_404_ERRORS) as NO_404_ERRORS,
        $udf(MISSING_FAVICON) as MISSING_FAVICON,
        $udf(NOT_MOBILE_FRIENDLY) as NOT_MOBILE_FRIENDLY,
        $udf(BIG_FAVICON_ABSENT) as BIG_FAVICON_ABSENT,
        $udf(MORDA_ERROR) as MORDA_ERROR,
        $udf(CONNECT_FAILED) as CONNECT_FAILED,
    from $extracted
);

$report = (
    select 
        ReportDate,
        count(*) as TotalHosts,

        count_if(DISALLOWED_IN_ROBOTS.HadProblem) as DISALLOWED_IN_ROBOTS_HadProblem,
        count_if(DISALLOWED_IN_ROBOTS.Fixed) as DISALLOWED_IN_ROBOTS_Fixed,
        count_if(DISALLOWED_IN_ROBOTS.FixedAfterVisit) as DISALLOWED_IN_ROBOTS_FixedAfterVisit,
        count_if(DISALLOWED_IN_ROBOTS.FixedAfterVisitDiag) as DISALLOWED_IN_ROBOTS_FixedAfterVisitDiag,
        avg_if(DISALLOWED_IN_ROBOTS.ProblemFixedIn, DISALLOWED_IN_ROBOTS.Fixed) as DISALLOWED_IN_ROBOTS_FixedInAvg,
        Yson::Serialize(Yson::From(histogram(if(DISALLOWED_IN_ROBOTS.Fixed, DISALLOWED_IN_ROBOTS.ProblemFixedIn)))) as DISALLOWED_IN_ROBOTS_Hist,
        aggregate_by(if(DISALLOWED_IN_ROBOTS.Fixed, DISALLOWED_IN_ROBOTS.ProblemFixedIn), $agg_p75) as DISALLOWED_IN_ROBOTS_ProblemFixedInP75,
        aggregate_by(if(DISALLOWED_IN_ROBOTS.Fixed, DISALLOWED_IN_ROBOTS.ProblemFixedIn), $agg_p90) as DISALLOWED_IN_ROBOTS_ProblemFixedInP90,
    
        count_if(DNS_ERROR.HadProblem) as DNS_ERROR_HadProblem,
        count_if(DNS_ERROR.Fixed) as DNS_ERROR_Fixed,
        count_if(DNS_ERROR.FixedAfterVisit) as DNS_ERROR_FixedAfterVisit,
        count_if(DNS_ERROR.FixedAfterVisitDiag) as DNS_ERROR_FixedAfterVisitDiag,
        avg_if(DNS_ERROR.ProblemFixedIn, DNS_ERROR.Fixed) as DNS_ERROR_FixedInAvg,
        Yson::Serialize(Yson::From(histogram(if(DNS_ERROR.Fixed, DNS_ERROR.ProblemFixedIn)))) as DNS_ERROR_Hist,
        aggregate_by(if(DNS_ERROR.Fixed, DNS_ERROR.ProblemFixedIn), $agg_p75) as DNS_ERROR_ProblemFixedInP75,
        aggregate_by(if(DNS_ERROR.Fixed, DNS_ERROR.ProblemFixedIn), $agg_p90) as DNS_ERROR_ProblemFixedInP90,
    
        count_if(SSL_CERTIFICATE_ERROR.HadProblem) as SSL_CERTIFICATE_ERROR_HadProblem,
        count_if(SSL_CERTIFICATE_ERROR.Fixed) as SSL_CERTIFICATE_ERROR_Fixed,
        count_if(SSL_CERTIFICATE_ERROR.FixedAfterVisit) as SSL_CERTIFICATE_ERROR_FixedAfterVisit,
        count_if(SSL_CERTIFICATE_ERROR.FixedAfterVisitDiag) as SSL_CERTIFICATE_ERROR_FixedAfterVisitDiag,
        avg_if(SSL_CERTIFICATE_ERROR.ProblemFixedIn, SSL_CERTIFICATE_ERROR.Fixed) as SSL_CERTIFICATE_ERROR_FixedInAvg,
        Yson::Serialize(Yson::From(histogram(if(SSL_CERTIFICATE_ERROR.Fixed, SSL_CERTIFICATE_ERROR.ProblemFixedIn)))) as SSL_CERTIFICATE_ERROR_Hist,
        aggregate_by(if(SSL_CERTIFICATE_ERROR.Fixed, SSL_CERTIFICATE_ERROR.ProblemFixedIn), $agg_p75) as SSL_CERTIFICATE_ERROR_ProblemFixedInP75,
        aggregate_by(if(SSL_CERTIFICATE_ERROR.Fixed, SSL_CERTIFICATE_ERROR.ProblemFixedIn), $agg_p90) as SSL_CERTIFICATE_ERROR_ProblemFixedInP90,
    
        count_if(TOO_MANY_BROKEN_LINKS.HadProblem) as TOO_MANY_BROKEN_LINKS_HadProblem,
        count_if(TOO_MANY_BROKEN_LINKS.Fixed) as TOO_MANY_BROKEN_LINKS_Fixed,
        count_if(TOO_MANY_BROKEN_LINKS.FixedAfterVisit) as TOO_MANY_BROKEN_LINKS_FixedAfterVisit,
        count_if(TOO_MANY_BROKEN_LINKS.FixedAfterVisitDiag) as TOO_MANY_BROKEN_LINKS_FixedAfterVisitDiag,
        avg_if(TOO_MANY_BROKEN_LINKS.ProblemFixedIn, TOO_MANY_BROKEN_LINKS.Fixed) as TOO_MANY_BROKEN_LINKS_FixedInAvg,
        Yson::Serialize(Yson::From(histogram(if(TOO_MANY_BROKEN_LINKS.Fixed, TOO_MANY_BROKEN_LINKS.ProblemFixedIn)))) as TOO_MANY_BROKEN_LINKS_Hist,
        aggregate_by(if(TOO_MANY_BROKEN_LINKS.Fixed, TOO_MANY_BROKEN_LINKS.ProblemFixedIn), $agg_p75) as TOO_MANY_BROKEN_LINKS_ProblemFixedInP75,
        aggregate_by(if(TOO_MANY_BROKEN_LINKS.Fixed, TOO_MANY_BROKEN_LINKS.ProblemFixedIn), $agg_p90) as TOO_MANY_BROKEN_LINKS_ProblemFixedInP90,
    
        count_if(NO_METRIKA_COUNTER_BINDING.HadProblem) as NO_METRIKA_COUNTER_BINDING_HadProblem,
        count_if(NO_METRIKA_COUNTER_BINDING.Fixed) as NO_METRIKA_COUNTER_BINDING_Fixed,
        count_if(NO_METRIKA_COUNTER_BINDING.FixedAfterVisit) as NO_METRIKA_COUNTER_BINDING_FixedAfterVisit,
        count_if(NO_METRIKA_COUNTER_BINDING.FixedAfterVisitDiag) as NO_METRIKA_COUNTER_BINDING_FixedAfterVisitDiag,
        avg_if(NO_METRIKA_COUNTER_BINDING.ProblemFixedIn, NO_METRIKA_COUNTER_BINDING.Fixed) as NO_METRIKA_COUNTER_BINDING_FixedInAvg,
        Yson::Serialize(Yson::From(histogram(if(NO_METRIKA_COUNTER_BINDING.Fixed, NO_METRIKA_COUNTER_BINDING.ProblemFixedIn)))) as NO_METRIKA_COUNTER_BINDING_Hist,
        aggregate_by(if(NO_METRIKA_COUNTER_BINDING.Fixed, NO_METRIKA_COUNTER_BINDING.ProblemFixedIn), $agg_p75) as NO_METRIKA_COUNTER_BINDING_ProblemFixedInP75,
        aggregate_by(if(NO_METRIKA_COUNTER_BINDING.Fixed, NO_METRIKA_COUNTER_BINDING.ProblemFixedIn), $agg_p90) as NO_METRIKA_COUNTER_BINDING_ProblemFixedInP90,
    
        count_if(FAVICON_ERROR.HadProblem) as FAVICON_ERROR_HadProblem,
        count_if(FAVICON_ERROR.Fixed) as FAVICON_ERROR_Fixed,
        count_if(FAVICON_ERROR.FixedAfterVisit) as FAVICON_ERROR_FixedAfterVisit,
        count_if(FAVICON_ERROR.FixedAfterVisitDiag) as FAVICON_ERROR_FixedAfterVisitDiag,
        avg_if(FAVICON_ERROR.ProblemFixedIn, FAVICON_ERROR.Fixed) as FAVICON_ERROR_FixedInAvg,
        Yson::Serialize(Yson::From(histogram(if(FAVICON_ERROR.Fixed, FAVICON_ERROR.ProblemFixedIn)))) as FAVICON_ERROR_Hist,
        aggregate_by(if(FAVICON_ERROR.Fixed, FAVICON_ERROR.ProblemFixedIn), $agg_p75) as FAVICON_ERROR_ProblemFixedInP75,
        aggregate_by(if(FAVICON_ERROR.Fixed, FAVICON_ERROR.ProblemFixedIn), $agg_p90) as FAVICON_ERROR_ProblemFixedInP90,
    
        count_if(SITEMAP_NOT_SET.HadProblem) as SITEMAP_NOT_SET_HadProblem,
        count_if(SITEMAP_NOT_SET.Fixed) as SITEMAP_NOT_SET_Fixed,
        count_if(SITEMAP_NOT_SET.FixedAfterVisit) as SITEMAP_NOT_SET_FixedAfterVisit,
        count_if(SITEMAP_NOT_SET.FixedAfterVisitDiag) as SITEMAP_NOT_SET_FixedAfterVisitDiag,
        avg_if(SITEMAP_NOT_SET.ProblemFixedIn, SITEMAP_NOT_SET.Fixed) as SITEMAP_NOT_SET_FixedInAvg,
        Yson::Serialize(Yson::From(histogram(if(SITEMAP_NOT_SET.Fixed, SITEMAP_NOT_SET.ProblemFixedIn)))) as SITEMAP_NOT_SET_Hist,
        aggregate_by(if(SITEMAP_NOT_SET.Fixed, SITEMAP_NOT_SET.ProblemFixedIn), $agg_p75) as SITEMAP_NOT_SET_ProblemFixedInP75,
        aggregate_by(if(SITEMAP_NOT_SET.Fixed, SITEMAP_NOT_SET.ProblemFixedIn), $agg_p90) as SITEMAP_NOT_SET_ProblemFixedInP90,
    
        count_if(NO_METRIKA_COUNTER_CRAWL_ENABLED.HadProblem) as NO_METRIKA_COUNTER_CRAWL_ENABLED_HadProblem,
        count_if(NO_METRIKA_COUNTER_CRAWL_ENABLED.Fixed) as NO_METRIKA_COUNTER_CRAWL_ENABLED_Fixed,
        count_if(NO_METRIKA_COUNTER_CRAWL_ENABLED.FixedAfterVisit) as NO_METRIKA_COUNTER_CRAWL_ENABLED_FixedAfterVisit,
        count_if(NO_METRIKA_COUNTER_CRAWL_ENABLED.FixedAfterVisitDiag) as NO_METRIKA_COUNTER_CRAWL_ENABLED_FixedAfterVisitDiag,
        avg_if(NO_METRIKA_COUNTER_CRAWL_ENABLED.ProblemFixedIn, NO_METRIKA_COUNTER_CRAWL_ENABLED.Fixed) as NO_METRIKA_COUNTER_CRAWL_ENABLED_FixedInAvg,
        Yson::Serialize(Yson::From(histogram(if(NO_METRIKA_COUNTER_CRAWL_ENABLED.Fixed, NO_METRIKA_COUNTER_CRAWL_ENABLED.ProblemFixedIn)))) as NO_METRIKA_COUNTER_CRAWL_ENABLED_Hist,
        aggregate_by(if(NO_METRIKA_COUNTER_CRAWL_ENABLED.Fixed, NO_METRIKA_COUNTER_CRAWL_ENABLED.ProblemFixedIn), $agg_p75) as NO_METRIKA_COUNTER_CRAWL_ENABLED_ProblemFixedInP75,
        aggregate_by(if(NO_METRIKA_COUNTER_CRAWL_ENABLED.Fixed, NO_METRIKA_COUNTER_CRAWL_ENABLED.ProblemFixedIn), $agg_p90) as NO_METRIKA_COUNTER_CRAWL_ENABLED_ProblemFixedInP90,
    
        count_if(ERRORS_IN_SITEMAPS.HadProblem) as ERRORS_IN_SITEMAPS_HadProblem,
        count_if(ERRORS_IN_SITEMAPS.Fixed) as ERRORS_IN_SITEMAPS_Fixed,
        count_if(ERRORS_IN_SITEMAPS.FixedAfterVisit) as ERRORS_IN_SITEMAPS_FixedAfterVisit,
        count_if(ERRORS_IN_SITEMAPS.FixedAfterVisitDiag) as ERRORS_IN_SITEMAPS_FixedAfterVisitDiag,
        avg_if(ERRORS_IN_SITEMAPS.ProblemFixedIn, ERRORS_IN_SITEMAPS.Fixed) as ERRORS_IN_SITEMAPS_FixedInAvg,
        Yson::Serialize(Yson::From(histogram(if(ERRORS_IN_SITEMAPS.Fixed, ERRORS_IN_SITEMAPS.ProblemFixedIn)))) as ERRORS_IN_SITEMAPS_Hist,
        aggregate_by(if(ERRORS_IN_SITEMAPS.Fixed, ERRORS_IN_SITEMAPS.ProblemFixedIn), $agg_p75) as ERRORS_IN_SITEMAPS_ProblemFixedInP75,
        aggregate_by(if(ERRORS_IN_SITEMAPS.Fixed, ERRORS_IN_SITEMAPS.ProblemFixedIn), $agg_p90) as ERRORS_IN_SITEMAPS_ProblemFixedInP90,
    
        count_if(DOCUMENTS_MISSING_TITLE.HadProblem) as DOCUMENTS_MISSING_TITLE_HadProblem,
        count_if(DOCUMENTS_MISSING_TITLE.Fixed) as DOCUMENTS_MISSING_TITLE_Fixed,
        count_if(DOCUMENTS_MISSING_TITLE.FixedAfterVisit) as DOCUMENTS_MISSING_TITLE_FixedAfterVisit,
        count_if(DOCUMENTS_MISSING_TITLE.FixedAfterVisitDiag) as DOCUMENTS_MISSING_TITLE_FixedAfterVisitDiag,
        avg_if(DOCUMENTS_MISSING_TITLE.ProblemFixedIn, DOCUMENTS_MISSING_TITLE.Fixed) as DOCUMENTS_MISSING_TITLE_FixedInAvg,
        Yson::Serialize(Yson::From(histogram(if(DOCUMENTS_MISSING_TITLE.Fixed, DOCUMENTS_MISSING_TITLE.ProblemFixedIn)))) as DOCUMENTS_MISSING_TITLE_Hist,
        aggregate_by(if(DOCUMENTS_MISSING_TITLE.Fixed, DOCUMENTS_MISSING_TITLE.ProblemFixedIn), $agg_p75) as DOCUMENTS_MISSING_TITLE_ProblemFixedInP75,
        aggregate_by(if(DOCUMENTS_MISSING_TITLE.Fixed, DOCUMENTS_MISSING_TITLE.ProblemFixedIn), $agg_p90) as DOCUMENTS_MISSING_TITLE_ProblemFixedInP90,
    
        count_if(DOCUMENTS_MISSING_DESCRIPTION.HadProblem) as DOCUMENTS_MISSING_DESCRIPTION_HadProblem,
        count_if(DOCUMENTS_MISSING_DESCRIPTION.Fixed) as DOCUMENTS_MISSING_DESCRIPTION_Fixed,
        count_if(DOCUMENTS_MISSING_DESCRIPTION.FixedAfterVisit) as DOCUMENTS_MISSING_DESCRIPTION_FixedAfterVisit,
        count_if(DOCUMENTS_MISSING_DESCRIPTION.FixedAfterVisitDiag) as DOCUMENTS_MISSING_DESCRIPTION_FixedAfterVisitDiag,
        avg_if(DOCUMENTS_MISSING_DESCRIPTION.ProblemFixedIn, DOCUMENTS_MISSING_DESCRIPTION.Fixed) as DOCUMENTS_MISSING_DESCRIPTION_FixedInAvg,
        Yson::Serialize(Yson::From(histogram(if(DOCUMENTS_MISSING_DESCRIPTION.Fixed, DOCUMENTS_MISSING_DESCRIPTION.ProblemFixedIn)))) as DOCUMENTS_MISSING_DESCRIPTION_Hist,
        aggregate_by(if(DOCUMENTS_MISSING_DESCRIPTION.Fixed, DOCUMENTS_MISSING_DESCRIPTION.ProblemFixedIn), $agg_p75) as DOCUMENTS_MISSING_DESCRIPTION_ProblemFixedInP75,
        aggregate_by(if(DOCUMENTS_MISSING_DESCRIPTION.Fixed, DOCUMENTS_MISSING_DESCRIPTION.ProblemFixedIn), $agg_p90) as DOCUMENTS_MISSING_DESCRIPTION_ProblemFixedInP90,
    
        count_if(ROBOTS_TXT_ERROR.HadProblem) as ROBOTS_TXT_ERROR_HadProblem,
        count_if(ROBOTS_TXT_ERROR.Fixed) as ROBOTS_TXT_ERROR_Fixed,
        count_if(ROBOTS_TXT_ERROR.FixedAfterVisit) as ROBOTS_TXT_ERROR_FixedAfterVisit,
        count_if(ROBOTS_TXT_ERROR.FixedAfterVisitDiag) as ROBOTS_TXT_ERROR_FixedAfterVisitDiag,
        avg_if(ROBOTS_TXT_ERROR.ProblemFixedIn, ROBOTS_TXT_ERROR.Fixed) as ROBOTS_TXT_ERROR_FixedInAvg,
        Yson::Serialize(Yson::From(histogram(if(ROBOTS_TXT_ERROR.Fixed, ROBOTS_TXT_ERROR.ProblemFixedIn)))) as ROBOTS_TXT_ERROR_Hist,
        aggregate_by(if(ROBOTS_TXT_ERROR.Fixed, ROBOTS_TXT_ERROR.ProblemFixedIn), $agg_p75) as ROBOTS_TXT_ERROR_ProblemFixedInP75,
        aggregate_by(if(ROBOTS_TXT_ERROR.Fixed, ROBOTS_TXT_ERROR.ProblemFixedIn), $agg_p90) as ROBOTS_TXT_ERROR_ProblemFixedInP90,
    
        count_if(NO_ROBOTS_TXT.HadProblem) as NO_ROBOTS_TXT_HadProblem,
        count_if(NO_ROBOTS_TXT.Fixed) as NO_ROBOTS_TXT_Fixed,
        count_if(NO_ROBOTS_TXT.FixedAfterVisit) as NO_ROBOTS_TXT_FixedAfterVisit,
        count_if(NO_ROBOTS_TXT.FixedAfterVisitDiag) as NO_ROBOTS_TXT_FixedAfterVisitDiag,
        avg_if(NO_ROBOTS_TXT.ProblemFixedIn, NO_ROBOTS_TXT.Fixed) as NO_ROBOTS_TXT_FixedInAvg,
        Yson::Serialize(Yson::From(histogram(if(NO_ROBOTS_TXT.Fixed, NO_ROBOTS_TXT.ProblemFixedIn)))) as NO_ROBOTS_TXT_Hist,
        aggregate_by(if(NO_ROBOTS_TXT.Fixed, NO_ROBOTS_TXT.ProblemFixedIn), $agg_p75) as NO_ROBOTS_TXT_ProblemFixedInP75,
        aggregate_by(if(NO_ROBOTS_TXT.Fixed, NO_ROBOTS_TXT.ProblemFixedIn), $agg_p90) as NO_ROBOTS_TXT_ProblemFixedInP90,
    
        count_if(NO_404_ERRORS.HadProblem) as NO_404_ERRORS_HadProblem,
        count_if(NO_404_ERRORS.Fixed) as NO_404_ERRORS_Fixed,
        count_if(NO_404_ERRORS.FixedAfterVisit) as NO_404_ERRORS_FixedAfterVisit,
        count_if(NO_404_ERRORS.FixedAfterVisitDiag) as NO_404_ERRORS_FixedAfterVisitDiag,
        avg_if(NO_404_ERRORS.ProblemFixedIn, NO_404_ERRORS.Fixed) as NO_404_ERRORS_FixedInAvg,
        Yson::Serialize(Yson::From(histogram(if(NO_404_ERRORS.Fixed, NO_404_ERRORS.ProblemFixedIn)))) as NO_404_ERRORS_Hist,
        aggregate_by(if(NO_404_ERRORS.Fixed, NO_404_ERRORS.ProblemFixedIn), $agg_p75) as NO_404_ERRORS_ProblemFixedInP75,
        aggregate_by(if(NO_404_ERRORS.Fixed, NO_404_ERRORS.ProblemFixedIn), $agg_p90) as NO_404_ERRORS_ProblemFixedInP90,
    
        count_if(MISSING_FAVICON.HadProblem) as MISSING_FAVICON_HadProblem,
        count_if(MISSING_FAVICON.Fixed) as MISSING_FAVICON_Fixed,
        count_if(MISSING_FAVICON.FixedAfterVisit) as MISSING_FAVICON_FixedAfterVisit,
        count_if(MISSING_FAVICON.FixedAfterVisitDiag) as MISSING_FAVICON_FixedAfterVisitDiag,
        avg_if(MISSING_FAVICON.ProblemFixedIn, MISSING_FAVICON.Fixed) as MISSING_FAVICON_FixedInAvg,
        Yson::Serialize(Yson::From(histogram(if(MISSING_FAVICON.Fixed, MISSING_FAVICON.ProblemFixedIn)))) as MISSING_FAVICON_Hist,
        aggregate_by(if(MISSING_FAVICON.Fixed, MISSING_FAVICON.ProblemFixedIn), $agg_p75) as MISSING_FAVICON_ProblemFixedInP75,
        aggregate_by(if(MISSING_FAVICON.Fixed, MISSING_FAVICON.ProblemFixedIn), $agg_p90) as MISSING_FAVICON_ProblemFixedInP90,
    
        count_if(NOT_MOBILE_FRIENDLY.HadProblem) as NOT_MOBILE_FRIENDLY_HadProblem,
        count_if(NOT_MOBILE_FRIENDLY.Fixed) as NOT_MOBILE_FRIENDLY_Fixed,
        count_if(NOT_MOBILE_FRIENDLY.FixedAfterVisit) as NOT_MOBILE_FRIENDLY_FixedAfterVisit,
        count_if(NOT_MOBILE_FRIENDLY.FixedAfterVisitDiag) as NOT_MOBILE_FRIENDLY_FixedAfterVisitDiag,
        avg_if(NOT_MOBILE_FRIENDLY.ProblemFixedIn, NOT_MOBILE_FRIENDLY.Fixed) as NOT_MOBILE_FRIENDLY_FixedInAvg,
        Yson::Serialize(Yson::From(histogram(if(NOT_MOBILE_FRIENDLY.Fixed, NOT_MOBILE_FRIENDLY.ProblemFixedIn)))) as NOT_MOBILE_FRIENDLY_Hist,
        aggregate_by(if(NOT_MOBILE_FRIENDLY.Fixed, NOT_MOBILE_FRIENDLY.ProblemFixedIn), $agg_p75) as NOT_MOBILE_FRIENDLY_ProblemFixedInP75,
        aggregate_by(if(NOT_MOBILE_FRIENDLY.Fixed, NOT_MOBILE_FRIENDLY.ProblemFixedIn), $agg_p90) as NOT_MOBILE_FRIENDLY_ProblemFixedInP90,
    
        count_if(BIG_FAVICON_ABSENT.HadProblem) as BIG_FAVICON_ABSENT_HadProblem,
        count_if(BIG_FAVICON_ABSENT.Fixed) as BIG_FAVICON_ABSENT_Fixed,
        count_if(BIG_FAVICON_ABSENT.FixedAfterVisit) as BIG_FAVICON_ABSENT_FixedAfterVisit,
        count_if(BIG_FAVICON_ABSENT.FixedAfterVisitDiag) as BIG_FAVICON_ABSENT_FixedAfterVisitDiag,
        avg_if(BIG_FAVICON_ABSENT.ProblemFixedIn, BIG_FAVICON_ABSENT.Fixed) as BIG_FAVICON_ABSENT_FixedInAvg,
        Yson::Serialize(Yson::From(histogram(if(BIG_FAVICON_ABSENT.Fixed, BIG_FAVICON_ABSENT.ProblemFixedIn)))) as BIG_FAVICON_ABSENT_Hist,
        aggregate_by(if(BIG_FAVICON_ABSENT.Fixed, BIG_FAVICON_ABSENT.ProblemFixedIn), $agg_p75) as BIG_FAVICON_ABSENT_ProblemFixedInP75,
        aggregate_by(if(BIG_FAVICON_ABSENT.Fixed, BIG_FAVICON_ABSENT.ProblemFixedIn), $agg_p90) as BIG_FAVICON_ABSENT_ProblemFixedInP90,
    
        count_if(MORDA_ERROR.HadProblem) as MORDA_ERROR_HadProblem,
        count_if(MORDA_ERROR.Fixed) as MORDA_ERROR_Fixed,
        count_if(MORDA_ERROR.FixedAfterVisit) as MORDA_ERROR_FixedAfterVisit,
        count_if(MORDA_ERROR.FixedAfterVisitDiag) as MORDA_ERROR_FixedAfterVisitDiag,
        avg_if(MORDA_ERROR.ProblemFixedIn, MORDA_ERROR.Fixed) as MORDA_ERROR_FixedInAvg,
        Yson::Serialize(Yson::From(histogram(if(MORDA_ERROR.Fixed, MORDA_ERROR.ProblemFixedIn)))) as MORDA_ERROR_Hist,
        aggregate_by(if(MORDA_ERROR.Fixed, MORDA_ERROR.ProblemFixedIn), $agg_p75) as MORDA_ERROR_ProblemFixedInP75,
        aggregate_by(if(MORDA_ERROR.Fixed, MORDA_ERROR.ProblemFixedIn), $agg_p90) as MORDA_ERROR_ProblemFixedInP90,

        count_if(CONNECT_FAILED.HadProblem) as CONNECT_FAILED_HadProblem,
        count_if(CONNECT_FAILED.Fixed) as CONNECT_FAILED_Fixed,
        count_if(CONNECT_FAILED.FixedAfterVisit) as CONNECT_FAILED_FixedAfterVisit,
        count_if(CONNECT_FAILED.FixedAfterVisitDiag) as CONNECT_FAILED_FixedAfterVisitDiag,
        avg_if(CONNECT_FAILED.ProblemFixedIn, CONNECT_FAILED.Fixed) as CONNECT_FAILED_FixedInAvg,
        Yson::Serialize(Yson::From(histogram(if(CONNECT_FAILED.Fixed, CONNECT_FAILED.ProblemFixedIn)))) as CONNECT_FAILED_Hist,
        aggregate_by(if(CONNECT_FAILED.Fixed, CONNECT_FAILED.ProblemFixedIn), $agg_p75) as CONNECT_FAILED_ProblemFixedInP75,
        aggregate_by(if(CONNECT_FAILED.Fixed, CONNECT_FAILED.ProblemFixedIn), $agg_p90) as CONNECT_FAILED_ProblemFixedInP90,
    
    from (
        select
            ReportDate,
            $get_stat(DISALLOWED_IN_ROBOTS) as DISALLOWED_IN_ROBOTS,
            $get_stat(DNS_ERROR) as DNS_ERROR,
            $get_stat(SSL_CERTIFICATE_ERROR) as SSL_CERTIFICATE_ERROR,
            $get_stat(TOO_MANY_BROKEN_LINKS) as TOO_MANY_BROKEN_LINKS,
            $get_stat(NO_METRIKA_COUNTER_BINDING) as NO_METRIKA_COUNTER_BINDING,
            $get_stat(FAVICON_ERROR) as FAVICON_ERROR,
            $get_stat(SITEMAP_NOT_SET) as SITEMAP_NOT_SET,
            $get_stat(NO_METRIKA_COUNTER_CRAWL_ENABLED) as NO_METRIKA_COUNTER_CRAWL_ENABLED,
            $get_stat(ERRORS_IN_SITEMAPS) as ERRORS_IN_SITEMAPS,
            $get_stat(DOCUMENTS_MISSING_TITLE) as DOCUMENTS_MISSING_TITLE,
            $get_stat(DOCUMENTS_MISSING_DESCRIPTION) as DOCUMENTS_MISSING_DESCRIPTION,
            $get_stat(ROBOTS_TXT_ERROR) as ROBOTS_TXT_ERROR,
            $get_stat(NO_ROBOTS_TXT) as NO_ROBOTS_TXT,
            $get_stat(NO_404_ERRORS) as NO_404_ERRORS,
            $get_stat(MISSING_FAVICON) as MISSING_FAVICON,
            $get_stat(NOT_MOBILE_FRIENDLY) as NOT_MOBILE_FRIENDLY,
            $get_stat(BIG_FAVICON_ABSENT) as BIG_FAVICON_ABSENT,
            $get_stat(MORDA_ERROR) as MORDA_ERROR,
            $get_stat(CONNECT_FAILED) as CONNECT_FAILED,
        from $stats
    )
    group by ReportDate
);

insert into $output_visits
    with truncate
select *
from $visits
order by Host, Period;

insert into $output_source
    with truncate
select *
from $source
order by Host, Period;

insert into $output_history
    with truncate
select *
from $history
order by Host, ReportDate;

insert into $output_extracted
    with truncate
select *
from $extracted
order by Host, ReportDate;

insert into $output_stats
    with truncate
select *
from $stats
order by Host, ReportDate;

insert into $output_report_tail
    with truncate
select *
from $report
order by ReportDate;

commit;

insert into $output_report
    with truncate
select
    ReportDate,
    some(TotalHosts) as TotalHosts,
    some(DISALLOWED_IN_ROBOTS_HadProblem) as DISALLOWED_IN_ROBOTS_HadProblem,
    some(DISALLOWED_IN_ROBOTS_Fixed) as DISALLOWED_IN_ROBOTS_Fixed,
    some(DISALLOWED_IN_ROBOTS_FixedAfterVisit) as DISALLOWED_IN_ROBOTS_FixedAfterVisit,
    some(DISALLOWED_IN_ROBOTS_FixedAfterVisitDiag) as DISALLOWED_IN_ROBOTS_FixedAfterVisitDiag,
    some(DISALLOWED_IN_ROBOTS_FixedInAvg) as DISALLOWED_IN_ROBOTS_FixedInAvg,
    some(DISALLOWED_IN_ROBOTS_Hist) as DISALLOWED_IN_ROBOTS_Hist,
    some(DISALLOWED_IN_ROBOTS_ProblemFixedInP75) as DISALLOWED_IN_ROBOTS_ProblemFixedInP75,
    some(DISALLOWED_IN_ROBOTS_ProblemFixedInP90) as DISALLOWED_IN_ROBOTS_ProblemFixedInP90,

    some(DNS_ERROR_HadProblem) as DNS_ERROR_HadProblem,
    some(DNS_ERROR_Fixed) as DNS_ERROR_Fixed,
    some(DNS_ERROR_FixedAfterVisit) as DNS_ERROR_FixedAfterVisit,
    some(DNS_ERROR_FixedAfterVisitDiag) as DNS_ERROR_FixedAfterVisitDiag,
    some(DNS_ERROR_FixedInAvg) as DNS_ERROR_FixedInAvg,
    some(DNS_ERROR_Hist) as DNS_ERROR_Hist,
    some(DNS_ERROR_ProblemFixedInP75) as DNS_ERROR_ProblemFixedInP75,
    some(DNS_ERROR_ProblemFixedInP90) as DNS_ERROR_ProblemFixedInP90,

    some(SSL_CERTIFICATE_ERROR_HadProblem) as SSL_CERTIFICATE_ERROR_HadProblem,
    some(SSL_CERTIFICATE_ERROR_Fixed) as SSL_CERTIFICATE_ERROR_Fixed,
    some(SSL_CERTIFICATE_ERROR_FixedAfterVisit) as SSL_CERTIFICATE_ERROR_FixedAfterVisit,
    some(SSL_CERTIFICATE_ERROR_FixedAfterVisitDiag) as SSL_CERTIFICATE_ERROR_FixedAfterVisitDiag,
    some(SSL_CERTIFICATE_ERROR_FixedInAvg) as SSL_CERTIFICATE_ERROR_FixedInAvg,
    some(SSL_CERTIFICATE_ERROR_Hist) as SSL_CERTIFICATE_ERROR_Hist,
    some(SSL_CERTIFICATE_ERROR_ProblemFixedInP75) as SSL_CERTIFICATE_ERROR_ProblemFixedInP75,
    some(SSL_CERTIFICATE_ERROR_ProblemFixedInP90) as SSL_CERTIFICATE_ERROR_ProblemFixedInP90,

    some(TOO_MANY_BROKEN_LINKS_HadProblem) as TOO_MANY_BROKEN_LINKS_HadProblem,
    some(TOO_MANY_BROKEN_LINKS_Fixed) as TOO_MANY_BROKEN_LINKS_Fixed,
    some(TOO_MANY_BROKEN_LINKS_FixedAfterVisit) as TOO_MANY_BROKEN_LINKS_FixedAfterVisit,
    some(TOO_MANY_BROKEN_LINKS_FixedAfterVisitDiag) as TOO_MANY_BROKEN_LINKS_FixedAfterVisitDiag,
    some(TOO_MANY_BROKEN_LINKS_FixedInAvg) as TOO_MANY_BROKEN_LINKS_FixedInAvg,
    some(TOO_MANY_BROKEN_LINKS_Hist) as TOO_MANY_BROKEN_LINKS_Hist,
    some(TOO_MANY_BROKEN_LINKS_ProblemFixedInP75) as TOO_MANY_BROKEN_LINKS_ProblemFixedInP75,
    some(TOO_MANY_BROKEN_LINKS_ProblemFixedInP90) as TOO_MANY_BROKEN_LINKS_ProblemFixedInP90,

    some(NO_METRIKA_COUNTER_BINDING_HadProblem) as NO_METRIKA_COUNTER_BINDING_HadProblem,
    some(NO_METRIKA_COUNTER_BINDING_Fixed) as NO_METRIKA_COUNTER_BINDING_Fixed,
    some(NO_METRIKA_COUNTER_BINDING_FixedAfterVisit) as NO_METRIKA_COUNTER_BINDING_FixedAfterVisit,
    some(NO_METRIKA_COUNTER_BINDING_FixedAfterVisitDiag) as NO_METRIKA_COUNTER_BINDING_FixedAfterVisitDiag,
    some(NO_METRIKA_COUNTER_BINDING_FixedInAvg) as NO_METRIKA_COUNTER_BINDING_FixedInAvg,
    some(NO_METRIKA_COUNTER_BINDING_Hist) as NO_METRIKA_COUNTER_BINDING_Hist,
    some(NO_METRIKA_COUNTER_BINDING_ProblemFixedInP75) as NO_METRIKA_COUNTER_BINDING_ProblemFixedInP75,
    some(NO_METRIKA_COUNTER_BINDING_ProblemFixedInP90) as NO_METRIKA_COUNTER_BINDING_ProblemFixedInP90,

    some(FAVICON_ERROR_HadProblem) as FAVICON_ERROR_HadProblem,
    some(FAVICON_ERROR_Fixed) as FAVICON_ERROR_Fixed,
    some(FAVICON_ERROR_FixedAfterVisit) as FAVICON_ERROR_FixedAfterVisit,
    some(FAVICON_ERROR_FixedAfterVisitDiag) as FAVICON_ERROR_FixedAfterVisitDiag,
    some(FAVICON_ERROR_FixedInAvg) as FAVICON_ERROR_FixedInAvg,
    some(FAVICON_ERROR_Hist) as FAVICON_ERROR_Hist,
    some(FAVICON_ERROR_ProblemFixedInP75) as FAVICON_ERROR_ProblemFixedInP75,
    some(FAVICON_ERROR_ProblemFixedInP90) as FAVICON_ERROR_ProblemFixedInP90,

    some(SITEMAP_NOT_SET_HadProblem) as SITEMAP_NOT_SET_HadProblem,
    some(SITEMAP_NOT_SET_Fixed) as SITEMAP_NOT_SET_Fixed,
    some(SITEMAP_NOT_SET_FixedAfterVisit) as SITEMAP_NOT_SET_FixedAfterVisit,
    some(SITEMAP_NOT_SET_FixedAfterVisitDiag) as SITEMAP_NOT_SET_FixedAfterVisitDiag,
    some(SITEMAP_NOT_SET_FixedInAvg) as SITEMAP_NOT_SET_FixedInAvg,
    some(SITEMAP_NOT_SET_Hist) as SITEMAP_NOT_SET_Hist,
    some(SITEMAP_NOT_SET_ProblemFixedInP75) as SITEMAP_NOT_SET_ProblemFixedInP75,
    some(SITEMAP_NOT_SET_ProblemFixedInP90) as SITEMAP_NOT_SET_ProblemFixedInP90,

    some(NO_METRIKA_COUNTER_CRAWL_ENABLED_HadProblem) as NO_METRIKA_COUNTER_CRAWL_ENABLED_HadProblem,
    some(NO_METRIKA_COUNTER_CRAWL_ENABLED_Fixed) as NO_METRIKA_COUNTER_CRAWL_ENABLED_Fixed,
    some(NO_METRIKA_COUNTER_CRAWL_ENABLED_FixedAfterVisit) as NO_METRIKA_COUNTER_CRAWL_ENABLED_FixedAfterVisit,
    some(NO_METRIKA_COUNTER_CRAWL_ENABLED_FixedAfterVisitDiag) as NO_METRIKA_COUNTER_CRAWL_ENABLED_FixedAfterVisitDiag,
    some(NO_METRIKA_COUNTER_CRAWL_ENABLED_FixedInAvg) as NO_METRIKA_COUNTER_CRAWL_ENABLED_FixedInAvg,
    some(NO_METRIKA_COUNTER_CRAWL_ENABLED_Hist) as NO_METRIKA_COUNTER_CRAWL_ENABLED_Hist,
    some(NO_METRIKA_COUNTER_CRAWL_ENABLED_ProblemFixedInP75) as NO_METRIKA_COUNTER_CRAWL_ENABLED_ProblemFixedInP75,
    some(NO_METRIKA_COUNTER_CRAWL_ENABLED_ProblemFixedInP90) as NO_METRIKA_COUNTER_CRAWL_ENABLED_ProblemFixedInP90,

    some(ERRORS_IN_SITEMAPS_HadProblem) as ERRORS_IN_SITEMAPS_HadProblem,
    some(ERRORS_IN_SITEMAPS_Fixed) as ERRORS_IN_SITEMAPS_Fixed,
    some(ERRORS_IN_SITEMAPS_FixedAfterVisit) as ERRORS_IN_SITEMAPS_FixedAfterVisit,
    some(ERRORS_IN_SITEMAPS_FixedAfterVisitDiag) as ERRORS_IN_SITEMAPS_FixedAfterVisitDiag,
    some(ERRORS_IN_SITEMAPS_FixedInAvg) as ERRORS_IN_SITEMAPS_FixedInAvg,
    some(ERRORS_IN_SITEMAPS_Hist) as ERRORS_IN_SITEMAPS_Hist,
    some(ERRORS_IN_SITEMAPS_ProblemFixedInP75) as ERRORS_IN_SITEMAPS_ProblemFixedInP75,
    some(ERRORS_IN_SITEMAPS_ProblemFixedInP90) as ERRORS_IN_SITEMAPS_ProblemFixedInP90,

    some(DOCUMENTS_MISSING_TITLE_HadProblem) as DOCUMENTS_MISSING_TITLE_HadProblem,
    some(DOCUMENTS_MISSING_TITLE_Fixed) as DOCUMENTS_MISSING_TITLE_Fixed,
    some(DOCUMENTS_MISSING_TITLE_FixedAfterVisit) as DOCUMENTS_MISSING_TITLE_FixedAfterVisit,
    some(DOCUMENTS_MISSING_TITLE_FixedAfterVisitDiag) as DOCUMENTS_MISSING_TITLE_FixedAfterVisitDiag,
    some(DOCUMENTS_MISSING_TITLE_FixedInAvg) as DOCUMENTS_MISSING_TITLE_FixedInAvg,
    some(DOCUMENTS_MISSING_TITLE_Hist) as DOCUMENTS_MISSING_TITLE_Hist,
    some(DOCUMENTS_MISSING_TITLE_ProblemFixedInP75) as DOCUMENTS_MISSING_TITLE_ProblemFixedInP75,
    some(DOCUMENTS_MISSING_TITLE_ProblemFixedInP90) as DOCUMENTS_MISSING_TITLE_ProblemFixedInP90,

    some(DOCUMENTS_MISSING_DESCRIPTION_HadProblem) as DOCUMENTS_MISSING_DESCRIPTION_HadProblem,
    some(DOCUMENTS_MISSING_DESCRIPTION_Fixed) as DOCUMENTS_MISSING_DESCRIPTION_Fixed,
    some(DOCUMENTS_MISSING_DESCRIPTION_FixedAfterVisit) as DOCUMENTS_MISSING_DESCRIPTION_FixedAfterVisit,
    some(DOCUMENTS_MISSING_DESCRIPTION_FixedAfterVisitDiag) as DOCUMENTS_MISSING_DESCRIPTION_FixedAfterVisitDiag,
    some(DOCUMENTS_MISSING_DESCRIPTION_FixedInAvg) as DOCUMENTS_MISSING_DESCRIPTION_FixedInAvg,
    some(DOCUMENTS_MISSING_DESCRIPTION_Hist) as DOCUMENTS_MISSING_DESCRIPTION_Hist,
    some(DOCUMENTS_MISSING_DESCRIPTION_ProblemFixedInP75) as DOCUMENTS_MISSING_DESCRIPTION_ProblemFixedInP75,
    some(DOCUMENTS_MISSING_DESCRIPTION_ProblemFixedInP90) as DOCUMENTS_MISSING_DESCRIPTION_ProblemFixedInP90,

    some(ROBOTS_TXT_ERROR_HadProblem) as ROBOTS_TXT_ERROR_HadProblem,
    some(ROBOTS_TXT_ERROR_Fixed) as ROBOTS_TXT_ERROR_Fixed,
    some(ROBOTS_TXT_ERROR_FixedAfterVisit) as ROBOTS_TXT_ERROR_FixedAfterVisit,
    some(ROBOTS_TXT_ERROR_FixedAfterVisitDiag) as ROBOTS_TXT_ERROR_FixedAfterVisitDiag,
    some(ROBOTS_TXT_ERROR_FixedInAvg) as ROBOTS_TXT_ERROR_FixedInAvg,
    some(ROBOTS_TXT_ERROR_Hist) as ROBOTS_TXT_ERROR_Hist,
    some(ROBOTS_TXT_ERROR_ProblemFixedInP75) as ROBOTS_TXT_ERROR_ProblemFixedInP75,
    some(ROBOTS_TXT_ERROR_ProblemFixedInP90) as ROBOTS_TXT_ERROR_ProblemFixedInP90,

    some(NO_ROBOTS_TXT_HadProblem) as NO_ROBOTS_TXT_HadProblem,
    some(NO_ROBOTS_TXT_Fixed) as NO_ROBOTS_TXT_Fixed,
    some(NO_ROBOTS_TXT_FixedAfterVisit) as NO_ROBOTS_TXT_FixedAfterVisit,
    some(NO_ROBOTS_TXT_FixedAfterVisitDiag) as NO_ROBOTS_TXT_FixedAfterVisitDiag,
    some(NO_ROBOTS_TXT_FixedInAvg) as NO_ROBOTS_TXT_FixedInAvg,
    some(NO_ROBOTS_TXT_Hist) as NO_ROBOTS_TXT_Hist,
    some(NO_ROBOTS_TXT_ProblemFixedInP75) as NO_ROBOTS_TXT_ProblemFixedInP75,
    some(NO_ROBOTS_TXT_ProblemFixedInP90) as NO_ROBOTS_TXT_ProblemFixedInP90,

    some(NO_404_ERRORS_HadProblem) as NO_404_ERRORS_HadProblem,
    some(NO_404_ERRORS_Fixed) as NO_404_ERRORS_Fixed,
    some(NO_404_ERRORS_FixedAfterVisit) as NO_404_ERRORS_FixedAfterVisit,
    some(NO_404_ERRORS_FixedAfterVisitDiag) as NO_404_ERRORS_FixedAfterVisitDiag,
    some(NO_404_ERRORS_FixedInAvg) as NO_404_ERRORS_FixedInAvg,
    some(NO_404_ERRORS_Hist) as NO_404_ERRORS_Hist,
    some(NO_404_ERRORS_ProblemFixedInP75) as NO_404_ERRORS_ProblemFixedInP75,
    some(NO_404_ERRORS_ProblemFixedInP90) as NO_404_ERRORS_ProblemFixedInP90,

    some(MISSING_FAVICON_HadProblem) as MISSING_FAVICON_HadProblem,
    some(MISSING_FAVICON_Fixed) as MISSING_FAVICON_Fixed,
    some(MISSING_FAVICON_FixedAfterVisit) as MISSING_FAVICON_FixedAfterVisit,
    some(MISSING_FAVICON_FixedAfterVisitDiag) as MISSING_FAVICON_FixedAfterVisitDiag,
    some(MISSING_FAVICON_FixedInAvg) as MISSING_FAVICON_FixedInAvg,
    some(MISSING_FAVICON_Hist) as MISSING_FAVICON_Hist,
    some(MISSING_FAVICON_ProblemFixedInP75) as MISSING_FAVICON_ProblemFixedInP75,
    some(MISSING_FAVICON_ProblemFixedInP90) as MISSING_FAVICON_ProblemFixedInP90,

    some(NOT_MOBILE_FRIENDLY_HadProblem) as NOT_MOBILE_FRIENDLY_HadProblem,
    some(NOT_MOBILE_FRIENDLY_Fixed) as NOT_MOBILE_FRIENDLY_Fixed,
    some(NOT_MOBILE_FRIENDLY_FixedAfterVisit) as NOT_MOBILE_FRIENDLY_FixedAfterVisit,
    some(NOT_MOBILE_FRIENDLY_FixedAfterVisitDiag) as NOT_MOBILE_FRIENDLY_FixedAfterVisitDiag,
    some(NOT_MOBILE_FRIENDLY_FixedInAvg) as NOT_MOBILE_FRIENDLY_FixedInAvg,
    some(NOT_MOBILE_FRIENDLY_Hist) as NOT_MOBILE_FRIENDLY_Hist,
    some(NOT_MOBILE_FRIENDLY_ProblemFixedInP75) as NOT_MOBILE_FRIENDLY_ProblemFixedInP75,
    some(NOT_MOBILE_FRIENDLY_ProblemFixedInP90) as NOT_MOBILE_FRIENDLY_ProblemFixedInP90,

    some(BIG_FAVICON_ABSENT_HadProblem) as BIG_FAVICON_ABSENT_HadProblem,
    some(BIG_FAVICON_ABSENT_Fixed) as BIG_FAVICON_ABSENT_Fixed,
    some(BIG_FAVICON_ABSENT_FixedAfterVisit) as BIG_FAVICON_ABSENT_FixedAfterVisit,
    some(BIG_FAVICON_ABSENT_FixedAfterVisitDiag) as BIG_FAVICON_ABSENT_FixedAfterVisitDiag,
    some(BIG_FAVICON_ABSENT_FixedInAvg) as BIG_FAVICON_ABSENT_FixedInAvg,
    some(BIG_FAVICON_ABSENT_Hist) as BIG_FAVICON_ABSENT_Hist,
    some(BIG_FAVICON_ABSENT_ProblemFixedInP75) as BIG_FAVICON_ABSENT_ProblemFixedInP75,
    some(BIG_FAVICON_ABSENT_ProblemFixedInP90) as BIG_FAVICON_ABSENT_ProblemFixedInP90,

    some(MORDA_ERROR_HadProblem) as MORDA_ERROR_HadProblem,
    some(MORDA_ERROR_Fixed) as MORDA_ERROR_Fixed,
    some(MORDA_ERROR_FixedAfterVisit) as MORDA_ERROR_FixedAfterVisit,
    some(MORDA_ERROR_FixedAfterVisitDiag) as MORDA_ERROR_FixedAfterVisitDiag,
    some(MORDA_ERROR_FixedInAvg) as MORDA_ERROR_FixedInAvg,
    some(MORDA_ERROR_Hist) as MORDA_ERROR_Hist,
    some(MORDA_ERROR_ProblemFixedInP75) as MORDA_ERROR_ProblemFixedInP75,
    some(MORDA_ERROR_ProblemFixedInP90) as MORDA_ERROR_ProblemFixedInP90,

    some(CONNECT_FAILED_HadProblem) as CONNECT_FAILED_HadProblem,
    some(CONNECT_FAILED_Fixed) as CONNECT_FAILED_Fixed,
    some(CONNECT_FAILED_FixedAfterVisit) as CONNECT_FAILED_FixedAfterVisit,
    some(CONNECT_FAILED_FixedAfterVisitDiag) as CONNECT_FAILED_FixedAfterVisitDiag,
    some(CONNECT_FAILED_FixedInAvg) as CONNECT_FAILED_FixedInAvg,
    some(CONNECT_FAILED_Hist) as CONNECT_FAILED_Hist,
    some(CONNECT_FAILED_ProblemFixedInP75) as CONNECT_FAILED_ProblemFixedInP75,
    some(CONNECT_FAILED_ProblemFixedInP90) as CONNECT_FAILED_ProblemFixedInP90,

from concat(
    $output_report,
    $output_report_tail
)
group by ReportDate
order by ReportDate;
