-- OPERANALYTICS-915
-- Дашборд про оплату за конверсии (action)
USE hahn;
PRAGMA yt.ParallelOperationsLimit = "7";  -- дефолтный лимит на пользователя 10
PRAGMA yt.DefaultOperationWeight = "0.5"; -- дефолт 1.0
PRAGMA yt.PoolTrees = "physical";
PRAGMA yt.TentativePoolTrees = "cloud";

$FIRST_TABLE = "2020-03-25";
$FORCE_UPDATE = Nothing(String?);  -- "YYYY-mm-dd" or Nothing(String?)
$SRC_FODLER = "//home/logfeller/logs/bs-action-checked-log/stream/1d";
$DST_FOLDER = "//home/vipplanners/dashboard_paid_actions/daily/action";
$SRC_FODLER_5MIN = "//home/logfeller/logs/bs-action-checked-log/stream/5min";
$DST_PATH_5MIN = "//home/vipplanners/dashboard_paid_actions/5min/action";
$TASK_MAX_TABLES = 30;  -- количество последних таблиц для обновления

DEFINE SUBQUERY $get_folder_tables($x, $first_table) AS
SELECT
  DateTime::MakeTimestamp(DateTime::ParseIso8601(Yson::LookupString(Attributes, "modification_time"))) AS table_mod_time
, Path AS table_path
, TableName(Path, "yt") AS table_name
FROM FOLDER($x, modification_time)
WHERE TableName(Path, "yt") >= $first_table
  AND Type == "table"
; END DEFINE;

$is_old_src_dst_mod_time = ($src_mt, $dst_mt) -> {
  RETURN ($dst_mt < $src_mt) OR ($dst_mt IS NULL);
};

-- 1. Берем последние $TASK_MAX_TABLES из источника
-- 2.а Оставляем только те, которые после $FORCE_UPDATE (при наличии) или 2.б
-- 2.б Оставляем только те, которых нет или были обновлены с последнего пересчета
$to_update = (
SELECT ListFlatMap(table_names, ($x) -> (IF($x.1, $x.0))) AS table_names
FROM (
  SELECT TOP(
      (src.table_name
      , IF(
          $FORCE_UPDATE IS NOT NULL
        , CAST($FORCE_UPDATE AS Date) <= CAST(src.table_name AS Date)
        , $is_old_src_dst_mod_time(src.table_mod_time, dst.table_mod_time))
      )
    , $TASK_MAX_TABLES
    ) AS table_names
  FROM $get_folder_tables($SRC_FODLER, $FIRST_TABLE) AS src
  LEFT JOIN $get_folder_tables($DST_FOLDER, $FIRST_TABLE) AS dst
    ON src.table_name == dst.table_name
));

DEFINE ACTION $process_table($date) AS
$DST_PATH = String::JoinFromList([$DST_FOLDER, $date], "/");
INSERT INTO $DST_PATH WITH TRUNCATE
SELECT
  CAST(orderid AS Int32) AS orderid
, eventdate
, is_desktop
, action_week
, $date AS table_date
, SUM(autobudgetavgcpa * sign / 1e6) / 1.18 * 30 AS tcost_rub_wo_nds  -- target cost
, SUM(actioncost * sign / 1e6) / 1.18 * 30 AS cost_rub_wo_nds  -- actual cost
, SUM(sign) AS actions
, SUM_IF(sign, actioncost = 0) AS actions_extra
FROM RANGE($SRC_FODLER, $date, $date)
WHERE isgoodconversion
  AND ListHas(ListMap(String::SplitToList(autobudgetoptions, ','), String::Strip), 'paid-actions')
GROUP BY
  orderid
, CAST(DateTime::MakeDate(DateTime::FromSeconds(CAST(eventtime AS Uint32))) AS String) AS eventdate
, MAX_OF(0, MIN_OF(4, DateTime::ToDays(DateTime::IntervalFromSeconds(CAST(goaleventtime AS Uint32) - CAST(eventtime AS Uint32))) / 7)) AS action_week
, devicetype >= 5 AS is_desktop
HAVING SUM(sign) > 0
ORDER BY
  orderid
, eventdate
, is_desktop
, action_week;
COMMIT;
END DEFINE;

EVALUATE FOR $x IN $to_update DO $process_table($x)
;

COMMIT;

INSERT INTO $DST_PATH_5MIN WITH TRUNCATE
SELECT
  CAST(orderid AS Int32) AS orderid
, eventdate
, is_desktop
, action_week
, SUM(autobudgetavgcpa * sign / 1e6) / 1.18 * 30 AS tcost_rub_wo_nds  -- target cost
, SUM(actioncost * sign / 1e6) / 1.18 * 30 AS cost_rub_wo_nds  -- actual cost
, SUM(sign) AS actions
, SUM_IF(sign, actioncost = 0) AS actions_extra
FROM RANGE($SRC_FODLER_5MIN)
WHERE isgoodconversion
  AND ListHas(ListMap(String::SplitToList(autobudgetoptions, ','), String::Strip), 'paid-actions')
GROUP BY
  orderid
, CAST(DateTime::MakeDate(DateTime::FromSeconds(CAST(eventtime AS Uint32))) AS String) AS eventdate
, MAX_OF(0, MIN_OF(4, DateTime::ToDays(DateTime::IntervalFromSeconds(CAST(goaleventtime AS Uint32) - CAST(eventtime AS Uint32))) / 7)) AS action_week
, devicetype >= 5 AS is_desktop
HAVING SUM(sign) > 0
ORDER BY
  orderid
, eventdate
, is_desktop
, action_week
;
