/* OPERANALYTICS-548
  Простои в двух частях
  1. Подневная статистика с днями без показов
  2. Подсчет метрик для дашборда

!ВНИМАНИЕ! counterparty = nvl(counterparty_id, client_id)
*/
PRAGMA yt.ParallelOperationsLimit = "7";  -- дефолтный лимит на пользователя 10
PRAGMA yt.DefaultOperationWeight = "0.5"; -- дефолт 1.0
PRAGMA yt.PoolTrees = "physical";
PRAGMA yt.TentativePoolTrees = "cloud";
USE hahn;

$DST_ROOT_FOLDER = "//home/vipplanners/dashboard_paused_clients";
$DST_CLIENT_D = $DST_ROOT_FOLDER || '/'  || "client_daily";
$DST_CLIENT_M = $DST_ROOT_FOLDER || '/'  || "client_monthly";
$DST_COUNTERPARTY_D = $DST_ROOT_FOLDER || '/'  || "counterparty_daily";
$DST_COUNTERPARTY_M = $DST_ROOT_FOLDER || '/'  || "counterparty_monthly";

-- Первая часть: подневная статистика с днями без показов
$DATE_START_SEC = EvaluateExpr(DateTime::ToSeconds(Date("2017-12-01")));

$order_stat = (
SELECT
  order_id
, date_day
, SUM(Cost * 1.) / 1e6 * 30 / 1.18 AS cost_rub_wo_nds
, SUM(Shows) AS shows
, SUM(Clicks) AS clicks
FROM `home/yabs/stat/OrderStatDay`
WHERE UpdateTime >= $DATE_START_SEC
  AND order_id > 0
GROUP BY
  OrderID AS order_id
, DateTime::MakeDate(
    DateTime::StartOfDay(
      DateTime::FromSeconds(
        Cast(UpdateTime + 3600 * 3 AS Uint32)))
  ) AS date_day
);

$direct_order = (
SELECT
  OrderID AS order_id
, ClientID as client_id
FROM `home/yabs/dict/path_to_dict_order_info_table`
WHERE EngineID = 7  -- Директ
  AND OrderType = 1  -- Коммерция
  AND OrderID > 0
  AND ClientID > 0
);

$client_stat = (
SELECT
  client_id
, date_day
, SUM(o.shows) ?? 0 AS shows
, SUM(o.clicks) ?? 0 AS clicks
, SUM(o.cost_rub_wo_nds) ?? 0 AS cost_rub_wo_nds
FROM $order_stat AS o
JOIN $direct_order AS d
  ON o.order_id == d.order_id
GROUP BY
  d.client_id AS client_id
, o.date_day AS date_day
);

$counterparty_stat = (
SELECT
  curr_counterparty_id
, date_day
, SUM(o.shows) ?? 0 AS shows
, SUM(o.clicks) ?? 0 AS clicks
, SUM(o.cost_rub_wo_nds) ?? 0 AS cost_rub_wo_nds
FROM $order_stat AS o
JOIN $direct_order AS d
  ON o.order_id == d.order_id
LEFT JOIN `//home/comdep-analytics/public/client_tiers/fact/latest_v2` AS crm
  ON d.client_id == crm.client_id
GROUP BY
  nvl(crm.curr_counterparty_id, d.client_id) AS curr_counterparty_id
, o.date_day AS date_day
);

$all_dates = (
SELECT DISTINCT date_day
FROM $client_stat
);

$all_client = (
SELECT DISTINCT client_id
FROM $client_stat
);

$all_counterparty = (
SELECT DISTINCT curr_counterparty_id
FROM $counterparty_stat
);

-- для фильтрации дней после старта показов 
$first_date_day_clients = (
SELECT client_id, MIN(date_day) AS first_date_day
FROM $client_stat
WHERE cost_rub_wo_nds > 0
GROUP BY client_id
);

-- для фильтрации дней после старта показов 
$first_date_day_counterparty = (
SELECT curr_counterparty_id, MIN(date_day) AS first_date_day
FROM $counterparty_stat
WHERE cost_rub_wo_nds > 0
GROUP BY curr_counterparty_id
);

$all_client_dates = (
SELECT
  d.date_day AS date_day
, c.client_id AS client_id
, fd.first_date_day AS first_date_day
FROM $all_dates AS d
CROSS JOIN $all_client AS c
JOIN $first_date_day_clients AS fd
  ON c.client_id == fd.client_id
WHERE
  d.date_day >= fd.first_date_day
);

$all_counterparty_dates = (
SELECT
  d.date_day AS date_day
, c.curr_counterparty_id AS curr_counterparty_id
, fd.first_date_day AS first_date_day
FROM $all_dates AS d
CROSS JOIN $all_counterparty AS c
JOIN $first_date_day_counterparty AS fd
  ON c.curr_counterparty_id == fd.curr_counterparty_id
WHERE
  d.date_day >= fd.first_date_day
);


INSERT INTO @client_daily_stat WITH TRUNCATE
SELECT
  idx.date_day AS date_day
, idx.client_id AS client_id
, idx.first_date_day AS first_date_day
, cs.shows ?? 0 AS shows
, cs.clicks ?? 0 AS clicks
, cs.cost_rub_wo_nds ?? 0 AS cost_rub_wo_nds
FROM $all_client_dates AS idx
LEFT JOIN $client_stat AS cs
  ON idx.date_day == cs.date_day
 AND idx.client_id == cs.client_id
ORDER BY client_id, date_day
;

INSERT INTO @counterparty_daily_stat WITH TRUNCATE
SELECT
  idx.date_day AS date_day
, idx.curr_counterparty_id AS curr_counterparty_id
, idx.first_date_day AS first_date_day
, cs.shows ?? 0 AS shows
, cs.clicks ?? 0 AS clicks
, cs.cost_rub_wo_nds ?? 0 AS cost_rub_wo_nds
FROM $all_counterparty_dates AS idx
LEFT JOIN $counterparty_stat AS cs
  ON idx.date_day == cs.date_day
 AND idx.curr_counterparty_id == cs.curr_counterparty_id
ORDER BY curr_counterparty_id, date_day
; COMMIT;


/* **************************************
Вторая часть: подсчет метрик для дашборда
  *************************************** */

$first_day = (
SELECT MIN(date_day) + Interval("P30D") FROM @client_daily_stat
);
$last_day = (
SELECT MAX(date_day) FROM @client_daily_stat
);
PRAGMA yt.DataSizePerJob = "50M";

INSERT INTO @client_daily_stat_jfp WITH TRUNCATE
SELECT
  client_id
, date_day
, SOME(c1.first_date_day) AS first_date_day
, SOME(c1.shows) AS shows
, SOME(c1.clicks) AS clicks
, SOME(c1.cost_rub_wo_nds) AS cost_rub_wo_nds
-- показатели из прошлого
, MAX(IF(
    c2.cost_rub_wo_nds > 0 AND
    (c1.date_day - c2.date_day BETWEEN Interval("P0D") AND Interval("P30D")),
    c2.date_day
  )) AS date_day_active_pred
-- показатели из будущего
, MIN(IF(
    c2.cost_rub_wo_nds > 0 AND
    (c2.date_day - c1.date_day BETWEEN Interval("P0D") AND Interval("P30D")),
    c2.date_day
  )) AS date_day_active_next
FROM @client_daily_stat AS c1
JOIN @client_daily_stat AS c2
  ON c1.client_id == c2.client_id
WHERE c1.date_day != c2.date_day 
  AND ABS(c1.date_day - c2.date_day) < Interval("P31D")
  AND c1.date_day > $first_day
  AND c1.date_day < $last_day
GROUP BY
  c1.client_id AS client_id
, c1.date_day AS date_day
;

INSERT INTO @counterparty_daily_stat_jfp WITH TRUNCATE
SELECT
  curr_counterparty_id
, date_day
, SOME(c1.first_date_day) AS first_date_day
, SOME(c1.shows) AS shows
, SOME(c1.clicks) AS clicks
, SOME(c1.cost_rub_wo_nds) AS cost_rub_wo_nds
-- показатели из прошлого
, MAX(IF(
    c2.cost_rub_wo_nds > 0 AND
    (c1.date_day - c2.date_day BETWEEN Interval("P0D") AND Interval("P30D")),
    c2.date_day
  )) AS date_day_active_pred
-- показатели из будущего
, MIN(IF(
    c2.cost_rub_wo_nds > 0 AND
    (c2.date_day - c1.date_day BETWEEN Interval("P0D") AND Interval("P30D")),
    c2.date_day
  )) AS date_day_active_next
FROM @counterparty_daily_stat AS c1
JOIN @counterparty_daily_stat AS c2
  ON c1.curr_counterparty_id == c2.curr_counterparty_id
WHERE c1.date_day != c2.date_day 
  AND ABS(c1.date_day - c2.date_day) < Interval("P31D")
  AND c1.date_day > $first_day
GROUP BY
  c1.curr_counterparty_id AS curr_counterparty_id
, c1.date_day AS date_day
; COMMIT;

PRAGMA yt.DataSizePerJob = default;

$client_cat = (
SELECT client_id, MAX_BY(category, cost) AS category
FROM `//home/comdep-analytics/ndorofeev/adhoc/ClientCustomgroup`
WHERE client_id > 0
GROUP BY client_id
);

$NOW_DATE = Cast(DateTime::MakeDate(DateTime::StartOfMonth(DateTime::Split(CurrentUtcDate()))) AS String);

-- список агентств с непросроченными договорами для Директа
$contract_agencies = (
SELECT agency_id, MAX_BY(contract_type_name, contract_id) AS contract_type_name
FROM `//home/balance/prod/bo/v_contract_apex_full`
WHERE
      ListHas(ListMap(String::SplitToList(ids_services, ','), ($x)->( Unwrap(Cast(String::Strip($x) AS Int64)) )), 7)  -- Директ
  AND NVL(finish_dt, '2222-01-01') >= $NOW_DATE
  AND contract_type IS NOT NULL
  AND agency_id > 0
GROUP BY agency_id
);

$counterparty_cat_tier = (
SELECT
  curr_counterparty_id
, category
, curr_client_tier
, client_type
, contract_type_name
, SUM(cat.cost) AS cost
FROM `//home/comdep-analytics/ndorofeev/adhoc/ClientCustomgroup` AS cat
LEFT JOIN `//home/comdep-analytics/public/client_tiers/fact/latest_v2` AS crm
  ON cat.client_id == crm.client_id
LEFT JOIN $contract_agencies AS ca
  ON crm.agency_id = ca.agency_id
WHERE cat.client_id > 0
GROUP BY
  nvl(crm.curr_counterparty_id, cat.client_id) AS curr_counterparty_id
, cat.category AS category
, NVL(crm.curr_counterparty_reporting_tier.name, "Unknown") AS curr_client_tier
, NVL(crm.client_type, "Unknown") AS client_type
, NVL(ca.contract_type_name, "Unknown") AS contract_type_name 
);
$counterparty_cat = (
SELECT curr_counterparty_id, MAX_BY(category, cost) AS category
FROM (
  SELECT curr_counterparty_id, category, SUM(cost) AS cost
  FROM $counterparty_cat_tier
  GROUP BY curr_counterparty_id, category
)
GROUP BY curr_counterparty_id
);
$counterparty_tier = (
SELECT curr_counterparty_id, MAX_BY(curr_client_tier, cost) AS curr_counterparty_tier
FROM (
  SELECT curr_counterparty_id, curr_client_tier, SUM(cost) AS cost
  FROM $counterparty_cat_tier
  GROUP BY curr_counterparty_id, curr_client_tier
)
GROUP BY curr_counterparty_id
);

$counterparty_type = (
SELECT curr_counterparty_id, MAX_BY(client_type, cost) AS curr_counterparty_type
FROM (
  SELECT curr_counterparty_id, client_type, SUM(cost) AS cost
  FROM $counterparty_cat_tier
  GROUP BY curr_counterparty_id, client_type
)
GROUP BY curr_counterparty_id
);

$counterparty_contract_type = (
SELECT curr_counterparty_id, MAX_BY(contract_type_name, cost) AS curr_counterparty_contract_type
FROM (
  SELECT curr_counterparty_id, contract_type_name, SUM(cost) AS cost
  FROM $counterparty_cat_tier
  GROUP BY curr_counterparty_id, contract_type_name
)
GROUP BY curr_counterparty_id
);

$crm_info_client = (
SELECT
  crm.client_id AS client_id
, crm.curr_counterparty_reporting_tier.name AS curr_client_tier
, crm.client_type AS client_type
, nvl(cc.category, "Unknown") AS curr_client_category
, nvl(ca.contract_type_name, "Unknown") AS curr_contract_type
FROM `//home/comdep-analytics/public/client_tiers/fact/latest_v2` AS crm
LEFT JOIN $client_cat AS cc
  ON crm.client_id == cc.client_id
LEFT JOIN $contract_agencies AS ca
  ON crm.agency_id = ca.agency_id
);

-- Простой это:
-- 1. в этот день не было показов
-- 2. в предыдущие 30 дней были открутки
-- 3. в последующие 30 дней случились открутки
$day_is_missed = ($row, $max_future_days) -> {
  -- $max_future_days -- размер окна для проверки активности
  $has_future_cost = NVL(
    $row.date_day_active_next - $row.date_day < DateTime::IntervalFromDays(CAST(1 + $max_future_days AS Int16)), False);
  $has_past_cost = NVL(
    $row.date_day - $row.date_day_active_pred < DateTime::IntervalFromDays(CAST(1 + $max_future_days AS Int16)), False);
  RETURN $row.shows == 0 AND $has_past_cost AND $has_future_cost;
};
$day_is_missed_wo_future = ($row) -> {
  $max_future_days = 30;  -- размер окна для проверки активности
  $has_past_cost = NVL(
    $row.date_day - $row.date_day_active_pred < DateTime::IntervalFromDays(CAST(1 + $max_future_days AS Int16)), False);
  RETURN $row.shows == 0 AND $has_past_cost;
};

INSERT INTO @client_daily_stat_final WITH TRUNCATE
SELECT
  cs.client_id AS client_id
, cs.date_day AS date_day
, DateTime::MakeDate(DateTime::StartOfMonth(cs.date_day)) AS date_month
, cs.shows AS shows
, cs.clicks AS clicks
, cs.cost_rub_wo_nds AS cost_rub_wo_nds
, cs.day_is_missed_3d AS day_is_missed_3d
, cs.day_is_missed_30d AS day_is_missed_30d
, cs.day_is_missed_wo_fut AS day_is_missed_wo_fut
, ci.curr_client_tier AS curr_client_tier
, ci.curr_client_category AS curr_client_category
, ci.client_type AS client_type
, ci.curr_contract_type AS curr_contract_type
FROM (
  SELECT
    client_id
  , date_day
  , shows
  , clicks
  , cost_rub_wo_nds
  , IF($day_is_missed(TableRow(), 3), 1, 0) AS day_is_missed_3d
  , IF($day_is_missed(TableRow(), 30), 1, 0) AS day_is_missed_30d
  , IF($day_is_missed_wo_future(TableRow()), 1, 0) AS day_is_missed_wo_fut
  FROM @client_daily_stat_jfp
) AS cs
JOIN $crm_info_client AS ci
  ON cs.client_id == ci.client_id
;

INSERT INTO @counterparty_daily_stat_final WITH TRUNCATE
SELECT
  cs.curr_counterparty_id AS curr_counterparty_id
, cs.date_day AS date_day
, DateTime::MakeDate(DateTime::StartOfMonth(cs.date_day)) AS date_month
, cs.shows AS shows
, cs.clicks AS clicks
, cs.cost_rub_wo_nds AS cost_rub_wo_nds
, cs.day_is_missed_3d AS day_is_missed_3d
, cs.day_is_missed_30d AS day_is_missed_30d
, cs.day_is_missed_wo_fut AS day_is_missed_wo_fut
, ci.category AS curr_counterparty_category
, ti.curr_counterparty_tier AS curr_counterparty_tier
, ct.curr_counterparty_type AS curr_counterparty_type
, cct.curr_counterparty_contract_type AS curr_counterparty_contract_type
FROM (
  SELECT
    curr_counterparty_id
  , date_day
  , shows
  , clicks
  , cost_rub_wo_nds
  , IF($day_is_missed(TableRow(), 3), 1, 0) AS day_is_missed_3d
  , IF($day_is_missed(TableRow(), 30), 1, 0) AS day_is_missed_30d
  , IF($day_is_missed_wo_future(TableRow()), 1, 0) AS day_is_missed_wo_fut
  FROM @counterparty_daily_stat_jfp
) AS cs
JOIN $counterparty_cat AS ci
  ON cs.curr_counterparty_id == ci.curr_counterparty_id
JOIN $counterparty_tier AS ti
  ON cs.curr_counterparty_id == ti.curr_counterparty_id
JOIN $counterparty_type AS ct
  ON cs.curr_counterparty_id == ct.curr_counterparty_id
JOIN $counterparty_contract_type AS cct
  ON cs.curr_counterparty_id == cct.curr_counterparty_id
; COMMIT;

-- подсчет потерянных дней и оценка ущерба для каждого клиента отдельно
$client_monthly = (
SELECT
  client_id
, date_month
-- cost / active_days -- средний чек по активным дням
, SUM(cost_rub_wo_nds) / COUNT_IF(shows > 0) AS avg_active_cost_rub_wo_nds
FROM @client_daily_stat_final
GROUP BY
  client_id
, date_month
HAVING SUM(shows) > 0
);

$counterparty_monthly = (
SELECT
  curr_counterparty_id
, date_month
-- cost / active_days -- средний чек по активным дням
, SUM(cost_rub_wo_nds) / COUNT_IF(shows > 0) AS avg_active_cost_rub_wo_nds
FROM @counterparty_daily_stat_final
GROUP BY
  curr_counterparty_id
, date_month
HAVING SUM(shows) > 0
);

INSERT INTO $DST_CLIENT_D WITH TRUNCATE
SELECT
  ds.client_id AS client_id
, ds.date_day AS date_day
, ds.date_month AS date_month
, DateTime::GetMonth(ds.date_month) AS num_month
, DateTime::GetYear(ds.date_month) AS num_year
, ds.shows AS shows
, ds.clicks AS clicks
, ds.cost_rub_wo_nds AS cost_rub_wo_nds
, ds.day_is_missed_3d AS day_is_missed_3d
, ds.day_is_missed_30d AS day_is_missed_30d
, ds.day_is_missed_wo_fut AS day_is_missed_wo_fut
, ds.curr_client_tier AS curr_client_tier
, ds.curr_client_category AS curr_client_category
, ds.client_type AS client_type
, ds.curr_contract_type AS curr_contract_type
, nvl(ms.avg_active_cost_rub_wo_nds, 0) AS avg_active_cost_rub_wo_nds
, nvl(ms.avg_active_cost_rub_wo_nds * ds.day_is_missed_30d, 0) AS lost_cost_30d
, nvl(ms.avg_active_cost_rub_wo_nds * ds.day_is_missed_3d, 0) AS lost_cost_3d
, nvl(ms.avg_active_cost_rub_wo_nds * ds.day_is_missed_wo_fut, 0) AS lost_cost_wo_fut
FROM @client_daily_stat_final AS ds
LEFT JOIN $client_monthly AS ms
   ON ds.client_id == ms.client_id
  AND ds.date_month == ms.date_month
ORDER BY client_id, date_day
;

INSERT INTO $DST_CLIENT_M WITH TRUNCATE
SELECT
  client_id
, date_month
, DateTime::GetMonth(date_month) AS num_month
, DateTime::GetYear(date_month) AS num_year
, SUM(shows) AS shows
, SUM(clicks) AS clicks
, SUM(cost_rub_wo_nds) AS cost_rub_wo_nds
, SUM(day_is_missed_3d) AS missed_days_3d
, SUM(day_is_missed_30d) AS missed_days_30d
, SUM(day_is_missed_wo_fut) AS missed_days_wo_fut
, SOME(curr_client_tier) AS curr_client_tier
, SOME(curr_client_category) AS curr_client_category
, SOME(client_type) AS client_type
, SOME(curr_contract_type) AS curr_contract_type
-- cost / active_days -- средний чек по активным дням
, IF(SUM(shows) > 0, SUM(cost_rub_wo_nds) / COUNT_IF(shows > 0), 0) AS avg_active_cost_rub_wo_nds
, nvl(IF(SUM(shows) > 0, SUM(cost_rub_wo_nds) / COUNT_IF(shows > 0), 0) * SUM(day_is_missed_30d), 0) AS lost_cost_30d
, nvl(IF(SUM(shows) > 0, SUM(cost_rub_wo_nds) / COUNT_IF(shows > 0), 0) * SUM(day_is_missed_3d), 0) AS lost_cost_3d
, nvl(IF(SUM(shows) > 0, SUM(cost_rub_wo_nds) / COUNT_IF(shows > 0), 0) * SUM(day_is_missed_wo_fut), 0) AS lost_cost_wo_fut
FROM @client_daily_stat_final
GROUP BY client_id, date_month
ORDER BY client_id, date_month
;


INSERT INTO $DST_COUNTERPARTY_D WITH TRUNCATE
SELECT
  ds.curr_counterparty_id AS curr_counterparty_id
, ds.date_day AS date_day
, ds.date_month AS date_month
, DateTime::GetMonth(ds.date_month) AS num_month
, DateTime::GetYear(ds.date_month) AS num_year
, ds.shows AS shows
, ds.clicks AS clicks
, ds.cost_rub_wo_nds AS cost_rub_wo_nds
, ds.day_is_missed_3d AS day_is_missed_3d
, ds.day_is_missed_30d AS day_is_missed_30d
, ds.day_is_missed_wo_fut AS day_is_missed_wo_fut
, ds.curr_counterparty_tier AS curr_counterparty_tier
, ds.curr_counterparty_category AS curr_counterparty_category
, ds.curr_counterparty_type AS curr_counterparty_type
, ds.curr_counterparty_contract_type AS curr_counterparty_contract_type
, nvl(ms.avg_active_cost_rub_wo_nds, 0) AS avg_active_cost_rub_wo_nds
, nvl(ms.avg_active_cost_rub_wo_nds * ds.day_is_missed_30d, 0) AS lost_cost_30d
, nvl(ms.avg_active_cost_rub_wo_nds * ds.day_is_missed_3d, 0) AS lost_cost_3d
, nvl(ms.avg_active_cost_rub_wo_nds * ds.day_is_missed_wo_fut, 0) AS lost_cost_wo_fut
FROM @counterparty_daily_stat_final AS ds
LEFT JOIN $counterparty_monthly AS ms
   ON ds.curr_counterparty_id == ms.curr_counterparty_id
  AND ds.date_month == ms.date_month
ORDER BY curr_counterparty_id, date_day
;

INSERT INTO $DST_COUNTERPARTY_M WITH TRUNCATE
SELECT
  curr_counterparty_id
, date_month
, DateTime::GetMonth(date_month) AS num_month
, DateTime::GetYear(date_month) AS num_year
, SUM(shows) AS shows
, SUM(clicks) AS clicks
, SUM(cost_rub_wo_nds) AS cost_rub_wo_nds
, SUM(day_is_missed_3d) AS missed_days_3d
, SUM(day_is_missed_30d) AS missed_days_30d
, SUM(day_is_missed_wo_fut) AS missed_days_wo_fut
, SOME(curr_counterparty_tier) AS curr_counterparty_tier
, SOME(curr_counterparty_category) AS curr_counterparty_category
, SOME(curr_counterparty_type) AS curr_counterparty_type
, SOME(curr_counterparty_contract_type) AS curr_counterparty_contract_type
-- cost / active_days -- средний чек по активным дням
, IF(SUM(shows) > 0, SUM(cost_rub_wo_nds) / COUNT_IF(shows > 0), 0) AS avg_active_cost_rub_wo_nds
, nvl(IF(SUM(shows) > 0, SUM(cost_rub_wo_nds) / COUNT_IF(shows > 0), 0) * SUM(day_is_missed_30d), 0) AS lost_cost_30d
, nvl(IF(SUM(shows) > 0, SUM(cost_rub_wo_nds) / COUNT_IF(shows > 0), 0) * SUM(day_is_missed_3d), 0) AS lost_cost_3d
, nvl(IF(SUM(shows) > 0, SUM(cost_rub_wo_nds) / COUNT_IF(shows > 0), 0) * SUM(day_is_missed_wo_fut), 0) AS lost_cost_wo_fut
FROM @counterparty_daily_stat_final
GROUP BY curr_counterparty_id, date_month
ORDER BY curr_counterparty_id, date_month
;