-- OPERANALYTICS-915
-- Дашборд про оплату за конверсии (chevent)
USE hahn;
PRAGMA yt.ParallelOperationsLimit = "7";  -- дефолтный лимит на пользователя 10
PRAGMA yt.DefaultOperationWeight = "0.5"; -- дефолт 1.0
PRAGMA yt.PoolTrees = "physical";
PRAGMA yt.TentativePoolTrees = "cloud";

$FIRST_TABLE = "2020-03-25";
$FORCE_UPDATE = Nothing(String?);  -- "YYYY-mm-dd" or Nothing(String?)
$SRC_FODLER = "//cooked_logs/bs-chevent-cooked-log/1d";
$DST_FOLDER = "//home/vipplanners/dashboard_paid_actions/daily/chevent";
$TASK_MAX_TABLES = 30;  -- количество последних таблиц для обновления

DEFINE SUBQUERY $get_folder_tables($x, $first_table) AS
SELECT
  DateTime::MakeTimestamp(DateTime::ParseIso8601(Yson::LookupString(Attributes, "modification_time"))) AS table_mod_time
, Path AS table_path
, TableName(Path, "yt") AS table_name
FROM FOLDER($x, modification_time)
WHERE TableName(Path, "yt") >= $first_table
  AND Type == "table"
; END DEFINE;

$is_old_src_dst_mod_time = ($src_mt, $dst_mt) -> {
  RETURN ($dst_mt < $src_mt) OR ($dst_mt IS NULL);
};

-- 1. Берем последние $TASK_MAX_TABLES из источника
-- 2.а Оставляем только те, которые после $FORCE_UPDATE (при наличии) или 2.б
-- 2.б Оставляем только те, которых нет или были обновлены с последнего пересчета
$to_update = (
SELECT ListFlatMap(table_names, ($x) -> (IF($x.1, $x.0))) AS table_names
FROM (
  SELECT TOP(
      (src.table_name
      , IF(
          $FORCE_UPDATE IS NOT NULL
        , CAST($FORCE_UPDATE AS Date) <= CAST(src.table_name AS Date)
        , $is_old_src_dst_mod_time(src.table_mod_time, dst.table_mod_time))
      )
    , $TASK_MAX_TABLES
    ) AS table_names
  FROM $get_folder_tables($SRC_FODLER, $FIRST_TABLE) AS src
  LEFT JOIN $get_folder_tables($DST_FOLDER, $FIRST_TABLE) AS dst
    ON src.table_name == dst.table_name
));

DEFINE ACTION $process_table($date) AS
$DST_PATH = String::JoinFromList([$DST_FOLDER, $date], "/");
INSERT INTO $DST_PATH WITH TRUNCATE
SELECT
  orderid
, eventdate
, is_desktop
, $date AS table_date
, SUM_IF(autobudgetavgcpa / 1e6, countertype = 1) / 1.18 * 30 AS tcost_rub_wo_nds  -- target cost
, COUNT_IF(countertype = 1) AS shows
, COUNT_IF(countertype = 2) AS clicks
, SUM(eventcost / 1e6) / 1.18 * 30 AS cost_rub_wo_nds
FROM RANGE($SRC_FODLER, $date, $date)
WHERE `autobudgetoptions_paid-actions`
  AND fraudbits = 0
  -- AND options_commerce  фильтр выключен специально, некоммерция рассматривается отдельно
GROUP BY
  orderid
, CAST(DateTime::MakeDate(DateTime::FromSeconds(CAST(eventtime AS Uint32))) AS String) AS eventdate
, devicetype >= 5 AS is_desktop
ORDER BY
  orderid
, eventdate
, is_desktop;
COMMIT;
END DEFINE;

EVALUATE FOR $x IN $to_update DO $process_table($x)
;
