-- OPERANALYTICS-536
-- Дашборд для смарт-баннеров (товарные фильтры)
-- !!без кампаний с оплатой за конверсии!!
PRAGMA yt.PoolTrees = "physical";
PRAGMA yt.TentativePoolTrees = "cloud";
PRAGMA yt.ParallelOperationsLimit = "7";
PRAGMA yt.DefaultOperationWeight = "0.5";
USE hahn;
$LAG_DAYS = 0;

$FIRST_TABLE = "2019-12-23";
$FORCE_UPDATE = Nothing(String?);  -- "YYYY-mm-dd" or Nothing(String?)
$TASK_MAX_TABLES = 100;  -- количество старейших таблиц для обновления за один запуск
$HISTORY_MAX_TABLES = 200;  -- сколько таблиц хранить
$SRC_FOLDER = "//cooked_logs/bs-chevent-cooked-log/1d";
$DST_FOLDER = "//home/vipplanners/dashboard_smartbanners/daily_v2/chevent";
$NOW = EvaluateExpr(CAST(CurrentUtcTimestamp() AS Timestamp));

DEFINE SUBQUERY $get_folder_tables($x, $first_table) AS
SELECT
  DateTime::MakeTimestamp(DateTime::ParseIso8601(Yson::LookupString(Attributes, "modification_time"))) AS table_mod_time
, Path AS table_path
, TableName(Path, "yt") AS table_name
FROM FOLDER($x, modification_time)
WHERE TableName(Path, "yt") >= $first_table
  AND Type == "table"
; END DEFINE;

$is_old_src_dst_mod_time = ($src_mt, $dst_mt) -> {
  RETURN $dst_mt < $src_mt OR $dst_mt IS NULL;
};

-- 1. Берем последние $TASK_MAX_TABLES из источника
-- 2.а Оставляем только те, которые после $FORCE_UPDATE (при наличии) или 2.б
-- 2.б Оставляем только те, которых нет или были обновлены с последнего пересчета
$to_update = (
SELECT ListFlatMap(table_names, ($x) -> (IF($x.1, $x.0))) AS table_names
FROM (
  SELECT TOP(
      (src.table_name
      , IF(
          $FORCE_UPDATE IS NOT NULL
        , CAST($FORCE_UPDATE AS Date) <= CAST(src.table_name AS Date)
        , $is_old_src_dst_mod_time(src.table_mod_time, dst.table_mod_time))
      )
    , $TASK_MAX_TABLES
    ) AS table_names
  FROM $get_folder_tables($SRC_FOLDER, $FIRST_TABLE) AS src
  LEFT JOIN $get_folder_tables($DST_FOLDER, $FIRST_TABLE) AS dst
    ON src.table_name == dst.table_name
));

INSERT INTO @dspcreative WITH TRUNCATE
SELECT
  CreativeID
, CreativeCodeID
, CreativeTemplateID
FROM `home/yabs/dict/DSPCreative`
;
COMMIT;

$classify_product = ($productype, $creativecodeid) -> {
  RETURN
    CASE
      WHEN $productype == 'direct' THEN 'smart_tgo'
      WHEN $productype == 'media-smart' AND $creativecodeid == 920 THEN 'smart_tile'
      WHEN $productype == 'media-smart' AND $creativecodeid != 920 THEN 'smart_banner'
      ELSE "other"
    END;
};
DEFINE ACTION $process_smartbanners($DATE) AS

$DST_PATH = String::JoinFromList([$DST_FOLDER, $DATE], "/");
$SRC_PATH = String::JoinFromList([$SRC_FOLDER, $DATE], "/");

INSERT INTO $DST_PATH WITH TRUNCATE
SELECT
  orderid
, pageid
, width
, height
, smart_type
, eventdate
, $DATE AS cheventtablename
, COUNT_IF(e.countertype == 1 AND e.bannerexportid != 0) AS shows_parent
, COUNT_IF(e.countertype == 1 AND e.bannerexportid == 0) AS shows_child
, COUNT_IF(e.countertype == 2) AS clicks
, SUM(e.eventcost / 1e6) / 1.18 * 30 AS cost_rub_wo_nds
FROM $SRC_PATH AS e
JOIN @dspcreative AS c
  ON e.creativeid == c.CreativeID
WHERE e.options_commerce
  AND NOT e.`autobudgetoptions_paid-actions`
  AND e.fraudbits == 0
  AND e.contexttype == 8  -- Товарные фильтры
  AND e.creativeid > 0
GROUP BY
  e.orderid AS orderid
, e.pageid AS pageid
, e.width AS width
, e.height AS height
, $classify_product(e.producttype, c.CreativeCodeID) AS smart_type
, CAST(DateTime::FromSeconds(CAST(eventtime AS Uint32) ?? 0) AS Date) AS eventdate
ORDER BY orderid, pageid;
COMMIT;
END DEFINE
;
EVALUATE FOR $x IN $to_update
DO $process_smartbanners($x);
COMMIT;

-- Удаление старых таблиц
$to_remove = (
SELECT AGGREGATE_LIST(Path)
FROM (
  SELECT Path, TableName(Path, "yt") AS table_name
  FROM FOLDER($DST_FOLDER)
  ORDER BY table_name DESC
  LIMIT 100000
  OFFSET $HISTORY_MAX_TABLES
  )
);
EVALUATE FOR $t IN $to_remove
DO BEGIN
  DROP TABLE $t; COMMIT;
END DO;
