USE hahn;

PRAGMA yt.MaxJobCount = "65536";
PRAGMA yt.DefaultMemoryLimit = "8G";
PRAGMA yt.DataSizePerJob = "50M";
PRAGMA yt.PoolTrees = "physical";
PRAGMA yt.TentativePoolTrees = "cloud";

/* READ ME 

Запрос предназначен для регулярного обновления данных для дашборда для поиска кейсов

Используемые данные: 
1) Директовый куб
2) Banners - Mediagroups
3) Маппинг клиентов

Ссылка на дашборд: https://dash.yandex-team.ru/0hvl4lk9lxqsy
Документация на Вики: https://wiki.yandex-team.ru/users/amatlakhov/Dashbord-dlja-poiska-kejjsov/

END */

$parse_date = DateTime::Parse("%Y-%m-%d");

---------------------- Ссылки на используемые таблицы -------------------------------------------
$stat_source         = "//home/comdep-cubes/direct/production/cube";
$bannerid_mediagroup = "//home/comdep-analytics/banner-categories/new/BannerIDMediaGroup";
$mediagroup_id       = "//home/comdep-analytics/chaos-ad/catalogia/mediagroups/latest";
$mapping             = "//home/comdep-analytics/common/BannerIDClient";
$meta                = "//home/vipplanners/dashboard_cases/meta";

---------------------- Путь для сохранения агрегата  --------------------------------------------
$project_path        = "//home/vipplanners/dashboard_cases/total";

---------------------- Таблицы для агрегатов по прод фокусам  -----------------------------------
$retargeting         = "//home/vipplanners/dashboard_cases/retargeting/all";
$dynamic             = "//home/vipplanners/dashboard_cases/dynamic/all";
$cpm_banner          = "//home/vipplanners/dashboard_cases/cpm_banner/all";
$mcb_banner          = "//home/vipplanners/dashboard_cases/mcb/all";
$image_ad            = "//home/vipplanners/dashboard_cases/image_ad/all";
$video_cpc           = "//home/vipplanners/dashboard_cases/video_cpc/all";
$smart_banners       = "//home/vipplanners/dashboard_cases/smart_banners/all";
$mobile_apps         = "//home/vipplanners/dashboard_cases/mobile_apps/all";
$total_network       = "//home/vipplanners/dashboard_cases/total_network/all";

----------------------- Словарь banner_type
$banner_type = ($x) -> {
RETURN 
CASE $x
    WHEN 'text'             THEN "Текстово-графические объявления"
    WHEN 'dynamic'          THEN "Динамические объявления"
    WHEN 'mcbanner'         THEN "МКБ"
    WHEN 'performance'      THEN "Смарт-баннеры"
    WHEN 'mobile_content'   THEN "Реклама мобильных приложений"
    WHEN 'cpm_banner'       THEN "Медийные баннеры"
    WHEN 'image_ad'         THEN "Графические объявления"
    WHEN 'cpc_video'        THEN "Видеообъявления"
    ELSE 'unknown_type'
END;
};

----------------------- Словарь product_type
$product_type = ($x) -> {
RETURN 
CASE $x
    WHEN 'video-motion'             THEN "Видеомоушн"
    WHEN 'video-creative-reach'     THEN "Медийное видео"
    WHEN 'video-creative'           THEN "Видеообъявления"
    WHEN 'auto-video-direct'        THEN "Видеодополнения"
    WHEN 'media-creative-reach'     THEN "ГО (охватный продукт)"
    WHEN 'media-creative'           THEN "ГО (конструктор)"
    WHEN 'media-image'              THEN "ГО (загруженные)"
    WHEN 'media-smart'              THEN "Смарт-баннеры"
    WHEN 'direct'                   THEN "ТГО"
    ELSE 'unknown_type'
END;
};

----------------------- Словарь context_type
$context_type = ($x) -> {
RETURN 
CASE $x
    WHEN 1     THEN "Фразы"
    WHEN 2     THEN "Ретаргетинг"
    WHEN 3     THEN "ДРФ"
    WHEN 5     THEN "Синоним"
    WHEN 7     THEN "ДО"
    WHEN 8     THEN "Товарные фильтры"
    WHEN 11    THEN "Автотаргетинг"
    ELSE 'unknown_type'
END;
};

---------------------- Словарь select_type 
$select_type = ($x) -> { RETURN IF($x  == 5, 'Поиск', 'Рся');}; -- Поиск, всё остальное - сеть

---------------------- Выделение ретаргетинга у смарт-баннеров
$retargeting_smart_banners =($x) -> {RETURN IF($x  == 27 OR $x  == 77, 
'Ретаргетинг (смарт-баннеры)', 'Новые пользователи (смарт-баннеры)');};


------------------------------------------------------------------------
--          Вычисление доступных таблиц для отчета                    --
------------------------------------------------------------------------

--- Максимальная дата в исходной папке куба директа
$max_date = (
    SELECT
        cast(max(dt) as Date) as max_date
    FROM (
        SELECT
            String::SplitToList(Path, '/')[5] as dt
        FROM
            FOLDER($stat_source)
        WHERE
            Type = "table"
        )
);

--- Максимальная дата в мета-таблице
$meta_date = 
(SELECT
        cast(max(fielddate) as Date) as fielddate
    FROM `//home/vipplanners/dashboard_cases/meta` WITH XLOCK
        );
        
--- Вычисление таблиц между датами
$make_is_during_last_run = ($meta_date, $max_date) -> {
  RETURN ($table_name) -> {
    $table_name = CAST($table_name AS Date);
    RETURN ($table_name BETWEEN $meta_date AND $max_date) ?? false;
  };
};

$is_during_last_run = $make_is_during_last_run(EvaluateExpr($meta_date  + Interval("P1D") ), EvaluateExpr($max_date));

------------------------------------------------------------------------
--          Экшн создания отчета                                      --
------------------------------------------------------------------------

DEFINE ACTION $making_datasets() AS

    --- Выгрузка статистики из куба
    $stat = (
    SELECT
        fielddate
    ,   client_id
    ,   cid
    ,   banner_id
    ,   select_type
    ,   context_type
    ,   banner_type
    ,   product_type
    ,   retargeting_smart_banners
    ,   SUM(shows)              AS shows
    ,   SUM(clicks)             AS clicks
    ,   SUM(cost_rub_wo_nds)    AS cost
    FROM FILTER($stat_source, $is_during_last_run)
    GROUP BY
        String::SplitToList(TablePath(), '/')[5] as fielddate
    ,   client_id
    ,   cid
    ,   banner_id
    ,   $select_type(select_type)               AS select_type
    ,   $context_type(context_type)             AS context_type
    ,   $banner_type(banner_type)               AS banner_type
    ,   $product_type(product_type)             AS product_type
    ,   $retargeting_smart_banners(select_type) AS retargeting_smart_banners
    );

    -- Прикрепление информации о медиагруппе
    $mediagroups = (
    SELECT 
        cat.BannerID as BannerID
    ,   medgr.mediagroup AS MediaGroup
    FROM
        $bannerid_mediagroup AS cat
    LEFT JOIN
        $mediagroup_id  AS medgr
    ON  CAST(cat.Category as string) = CAST(medgr.mediagroup_id as string)
    );

    -- Прикрепление информации о клиенте (если есть бренд из маппинга, если есть - домен, иначе - ClietID)
    $client_mapping = (
    SELECT
        BannerID
    ,   IF(Client = 'None', CAST(ClientID as string), Client) as Client
    FROM $mapping
    );
    
    ---- Собираем все в одну таблицу
    $total_finish = (
    SELECT
       fielddate
    ,   IF(DateTime::GetDayOfWeek($parse_date(fielddate)) == 6 OR 
        DateTime::GetDayOfWeek($parse_date(fielddate)) == 7, True, False) as is_weekend
    ,   client
    ,   'БЕЗ УЧЕТА КАТЕГОРИЗАЦИИ'   AS mediagroup
    ,   select_type
    ,   context_type
    ,   banner_type
    ,   product_type
    ,   retargeting_smart_banners
    ,   SUM(shows)                  AS shows
    ,   SUM(clicks)                 AS clicks
    ,   SUM(cost)                   AS cost
    FROM 
        $stat AS stat
    LEFT JOIN 
        $client_mapping AS cli
    ON stat.banner_id = cli.BannerID
    GROUP BY
        stat.fielddate                      AS fielddate
    ,   cli.Client??'Нет клиента'           AS client 
    ,   stat.select_type                    AS select_type
    ,   stat.context_type                   AS context_type 
    ,   stat.banner_type                    AS banner_type 
    ,   stat.product_type                   AS product_type
    ,   stat.retargeting_smart_banners      AS retargeting_smart_banners
    HAVING SUM(shows) > 0

    UNION ALL

    SELECT
        fielddate
    ,   IF(DateTime::GetDayOfWeek($parse_date(fielddate)) == 6 OR 
        DateTime::GetDayOfWeek($parse_date(fielddate)) == 7, True, False) as is_weekend
    ,   client
    ,   mediagroup
    ,   select_type
    ,   context_type
    ,   banner_type
    ,   product_type
    ,   retargeting_smart_banners
    ,   SUM(shows)                  AS shows
    ,   SUM(clicks)                 AS clicks
    ,   SUM(cost)                   AS cost
    FROM 
        $stat AS stat
    LEFT JOIN 
        $client_mapping AS cli
    ON stat.banner_id = cli.BannerID
    LEFT JOIN 
        $mediagroups AS mdg
    ON stat.banner_id = mdg.BannerID
    GROUP BY
        stat.fielddate                      AS fielddate
    ,   cli.Client??'Нет клиента'           AS client 
    ,   mdg.MediaGroup??'Нет медиагруппы'   AS mediagroup
    ,   stat.select_type                    AS select_type
    ,   stat.context_type                   AS context_type 
    ,   stat.banner_type                    AS banner_type 
    ,   stat.product_type                   AS product_type
    ,   stat.retargeting_smart_banners      AS retargeting_smart_banners
    HAVING SUM(shows) > 0);


    INSERT INTO $project_path
    SELECT *
    FROM $total_finish;
    
    COMMIT;
    
    ---- Разделение по продуктовым фокусам 
    INSERT INTO $retargeting
    SELECT 
        fielddate
    ,   is_weekend
    ,   client
    ,   mediagroup
    ,   SUM_IF(shows, context_type = "Ретаргетинг")             as shows_retargeting
    ,   SUM_IF(clicks, context_type = "Ретаргетинг")            as clicks_retargeting
    ,   SUM_IF(cost, context_type = "Ретаргетинг")              as cost_retargeting
    ,   SUM_IF(shows, select_type = 'Рся' AND context_type != "Ретаргетинг")            as network_shows_wo_retargeting
    ,   SUM_IF(clicks, select_type = 'Рся' AND context_type != "Ретаргетинг")           as network_clicks_wo_retargeting
    ,   SUM_IF(cost, select_type = 'Рся' AND context_type != "Ретаргетинг")             as network_cost_wo_retargeting
    ,   SUM(shows)   as shows
    ,   SUM(clicks)  as clicks
    ,   SUM(cost)    as cost
    FROM
        $total_finish
    GROUP BY
        fielddate
    ,   is_weekend
    ,   client
    ,   mediagroup
    HAVING 
        SUM_IF(shows, context_type = "Ретаргетинг") > 0;

    COMMIT;

    INSERT INTO $dynamic
    SELECT 
        fielddate
    ,   is_weekend
    ,   client
    ,   mediagroup
    ,   SUM_IF(shows, banner_type = "Динамические объявления")             as shows_DO
    ,   SUM_IF(clicks, banner_type = "Динамические объявления")            as clicks_DO
    ,   SUM_IF(cost, banner_type = "Динамические объявления")              as cost_DO
    ,   SUM_IF(shows, banner_type != "Динамические объявления")            as shows_wo_DO
    ,   SUM_IF(clicks, banner_type != "Динамические объявления")           as clicks_wo_DO
    ,   SUM_IF(cost, banner_type != "Динамические объявления")             as cost_wo_DO
    ,   SUM(shows)   as shows
    ,   SUM(clicks)  as clicks
    ,   SUM(cost)    as cost
    FROM
        $total_finish
    GROUP BY
        fielddate
    ,   is_weekend
    ,   client
    ,   mediagroup
    HAVING 
        SUM_IF(shows, banner_type = "Динамические объявления") > 0;

    COMMIT;
    
    INSERT INTO $cpm_banner
    SELECT 
        fielddate
    ,   is_weekend
    ,   client
    ,   mediagroup
    ,   SUM_IF(shows, banner_type = "Медийные баннеры")             as shows_cpm
    ,   SUM_IF(clicks, banner_type = "Медийные баннеры")            as clicks_cpm
    ,   SUM_IF(cost, banner_type = "Медийные баннеры")              as cost_cpm
    ,   SUM_IF(shows, select_type = 'Рся' AND banner_type != "Медийные баннеры")             as network_shows_wo_cpm
    ,   SUM_IF(clicks, select_type = 'Рся' AND banner_type != "Медийные баннеры")            as network_clicks_wo_cpm
    ,   SUM_IF(cost, select_type = 'Рся' AND banner_type != "Медийные баннеры")              as network_cost_wo_cpm
    ,   SUM(shows)   as shows
    ,   SUM(clicks)  as clicks
    ,   SUM(cost)    as cost
    FROM
        $total_finish
    GROUP BY
        fielddate
    ,   is_weekend
    ,   client
    ,   mediagroup
    HAVING 
       SUM_IF(shows, banner_type = "Медийные баннеры") > 0;
    
    COMMIT;
    
    INSERT INTO $mcb_banner
    SELECT 
        fielddate
    ,   is_weekend
    ,   client
    ,   mediagroup
    ,   SUM_IF(shows, banner_type = "МКБ")             as shows_mcb
    ,   SUM_IF(clicks, banner_type = "МКБ")            as clicks_mcb
    ,   SUM_IF(cost, banner_type = "МКБ")              as cost_mcb
    ,   SUM_IF(shows, banner_type != "МКБ")            as shows_wo_mcb
    ,   SUM_IF(clicks, banner_type != "МКБ")           as clicks_wo_mcb
    ,   SUM_IF(cost, banner_type != "МКБ")             as cost_wo_mcb
    ,   SUM(shows)   as shows
    ,   SUM(clicks)  as clicks
    ,   SUM(cost)    as cost
    FROM
        $total_finish
    GROUP BY
        fielddate
    ,   is_weekend
    ,   client
    ,   mediagroup
    HAVING
        SUM_IF(shows, banner_type = "МКБ") > 0;

    COMMIT;

    INSERT INTO $image_ad
    SELECT 
        fielddate
    ,   is_weekend
    ,   client
    ,   mediagroup
    ,   SUM_IF(shows, banner_type = "Графические объявления")             as shows_image_ad
    ,   SUM_IF(clicks, banner_type = "Графические объявления")            as clicks_image_ad
    ,   SUM_IF(cost, banner_type = "Графические объявления")              as cost_image_ad
    ,   SUM_IF(shows, select_type = 'Рся' AND banner_type != "Графические объявления")   as network_shows_wo_image_ad
    ,   SUM_IF(clicks, select_type = 'Рся' AND banner_type != "Графические объявления")  as network_clicks_wo_image_ad
    ,   SUM_IF(cost, select_type = 'Рся' AND banner_type != "Графические объявления")    as network_cost_wo_image_ad
    ,   SUM(shows)   as shows
    ,   SUM(clicks)  as clicks
    ,   SUM(cost)    as cost
    FROM
        $total_finish 
    GROUP BY
        fielddate
    ,   is_weekend
    ,   client
    ,   mediagroup
    HAVING 
        SUM_IF(shows, banner_type = "Графические объявления") > 0;
    
    COMMIT;

    INSERT INTO $video_cpc
    SELECT 
        fielddate
    ,   is_weekend
    ,   client
    ,   mediagroup
    ,   SUM_IF(shows, product_type = "Видеомоушн" OR product_type = "Видеообъявления" 
                OR product_type = "Видеодополнения")               as shows_video_cpc
    ,   SUM_IF(clicks, product_type = "Видеомоушн" OR product_type = "Видеообъявления" 
                OR product_type = "Видеодополнения")               as clicks_video_cpc
    ,   SUM_IF(cost, product_type = "Видеомоушн" OR product_type = "Видеообъявления" 
                OR product_type = "Видеодополнения")               as cost_video_cpc
    ,   SUM_IF(shows, select_type = 'Рся' AND product_type != "Видеомоушн" AND product_type != "Видеообъявления" 
                AND product_type != "Видеодополнения")             as network_shows_wo_video_cpc
    ,   SUM_IF(clicks, select_type = 'Рся' AND product_type != "Видеомоушн" AND product_type != "Видеообъявления" 
                AND product_type != "Видеодополнения")             as network_clicks_wo_video_cpc
    ,   SUM_IF(cost, select_type = 'Рся' AND product_type != "Видеомоушн" AND product_type != "Видеообъявления" 
                AND product_type != "Видеодополнения")             as network_cost_wo_video_cpc
    ,   SUM(shows)   as shows
    ,   SUM(clicks)  as clicks
    ,   SUM(cost)    as cost
    FROM
        $total_finish 
    GROUP BY
        fielddate
    ,   is_weekend
    ,   client
    ,   mediagroup
    HAVING
        SUM_IF(shows, product_type = "Видеомоушн" OR product_type = "Видеообъявления" 
                  OR product_type = "Видеодополнения") > 0;

    COMMIT;

    INSERT INTO $smart_banners
    SELECT 
        fielddate
    ,   is_weekend
    ,   client
    ,   mediagroup
    ,   retargeting_smart_banners
    ,   SUM_IF(shows, banner_type = "Смарт-баннеры")              as shows_smart
    ,   SUM_IF(clicks, banner_type = "Смарт-баннеры")             as clicks_smart
    ,   SUM_IF(cost, banner_type = "Смарт-баннеры")               as cost_smart
    ,   SUM_IF(shows, banner_type != "Смарт-баннеры")             as shows_wo_smart
    ,   SUM_IF(clicks, banner_type != "Смарт-баннеры")            as clicks_wo_smart
    ,   SUM_IF(cost, banner_type != "Смарт-баннеры")              as cost_wo_smart
    ,   SUM(shows)   as shows
    ,   SUM(clicks)  as clicks
    ,   SUM(cost)    as cost
    FROM
        $total_finish
    GROUP BY
        fielddate
    ,   is_weekend
    ,   client
    ,   mediagroup
    ,   retargeting_smart_banners
    HAVING 
        SUM_IF(shows, banner_type = "Смарт-баннеры") > 0;

    COMMIT;

    INSERT INTO $mobile_apps 
    SELECT 
        fielddate
    ,   is_weekend
    ,   client
    ,   mediagroup
    ,   SUM_IF(shows, banner_type = "Реклама мобильных приложений")              as shows_mobile_apps
    ,   SUM_IF(clicks, banner_type = "Реклама мобильных приложений")             as clicks_mobile_apps
    ,   SUM_IF(cost, banner_type = "Реклама мобильных приложений")               as cost_mobile_apps
    ,   SUM_IF(shows, banner_type != "Реклама мобильных приложений")             as shows_wo_mobile_apps
    ,   SUM_IF(clicks, banner_type != "Реклама мобильных приложений")            as clicks_wo_mobile_apps
    ,   SUM_IF(cost, banner_type != "Реклама мобильных приложений")              as cost_wo_mobile_apps
    ,   SUM(shows)   as shows
    ,   SUM(clicks)  as clicks
    ,   SUM(cost)    as cost
    FROM
        $total_finish 
    GROUP BY
        fielddate
    ,   is_weekend
    ,   client
    ,   mediagroup
    HAVING 
        SUM_IF(shows, banner_type = "Реклама мобильных приложений") > 0;

    COMMIT;

    INSERT INTO $total_network 
    SELECT 
        fielddate
    ,   is_weekend
    ,   client
    ,   mediagroup
    ,   SUM_IF(shows, select_type = "Рся")              as shows_network
    ,   SUM_IF(clicks, select_type = "Рся")             as clicks_network
    ,   SUM_IF(cost, select_type = "Рся")               as cost_network
    ,   SUM(shows)   as shows
    ,   SUM(clicks)  as clicks
    ,   SUM(cost)    as cost
    FROM
        $total_finish
    GROUP BY
        fielddate
    ,   is_weekend
    ,   client
    ,   mediagroup
    HAVING
        SUM_IF(shows, select_type = "Рся") > 0;

    COMMIT;
    
    INSERT INTO $meta
    SELECT CAST(max_date as string) as fielddate
    FROM $max_date;
    
    COMMIT;
    
END DEFINE;

EVALUATE IF $meta_date != $max_date DO $making_datasets();
COMMIT;
