USE hahn;

PRAGMA yt.QueryCacheMode = "normal";
PRAGMA yt.QueryCacheIgnoreTableRevision = "true";
PRAGMA yt.DefaultMaxJobFails = "15";
PRAGMA yt.ParallelOperationsLimit = "4";

DECLARE $TASK_AUTHOR_LOGIN AS String;
DECLARE $YT_BASE_PATH AS String;
DECLARE $YT_TASK_FOLDER AS String;
DECLARE $LOGINS AS List<String>;
DECLARE $DATE_BEGIN AS String;
DECLARE $DATE_FINISH AS String;
DECLARE $CAMPAIGN_IDS_WL AS List<Uint64>;

-- нужно только при вычислении агрегатов
$MAKE_ARCHIVE = False;
$YT_ARCHIVE_FOLDER = "archive";

/* Определение параметров для задачи расчета предагрегатов за 30 дней */
PRAGMA yt.DefaultOperationWeight = "0.5";

$RUN_AS_TEST = {{param[run_as_test]}};  -- True, False
$YT_TEST_FOLDER = "test";
$MAKE_ARCHIVE = EvaluateExpr(IF($RUN_AS_TEST, False, True));

$INTERVAL_DAYS = EvaluateExpr(IF($RUN_AS_TEST, Interval("P0D"), Interval("P30D")));
$NOW = EvaluateExpr(CurrentUtcDate());
$DATE_FINISH = EvaluateExpr(Unwrap(Cast(($NOW - Interval("P3D")) AS String)));
$DATE_BEGIN = EvaluateExpr(Unwrap(Cast(($NOW - Interval("P3D") - $INTERVAL_DAYS) AS String)));

$YT_BASE_PATH = "//home/vipplanners/ta_report";
$YT_TASK_FOLDER = EvaluateExpr(IF($RUN_AS_TEST, $YT_TEST_FOLDER, "30d"));

$TASK_AUTHOR_LOGIN = "robot-keywords";
$LOGINS = ListCreate(String);
$CAMPAIGN_IDS_WL = ListCreate(Uint64);
/* Конец параметров */


$YT_REPORT_TYPE = "search_queries";
$QUERY_VERSION = (0, 1, 2);

PRAGMA yt.Owners = "n-bar leroy";
PRAGMA yt.Pool = $TASK_AUTHOR_LOGIN;

$TODO = @@
- добавить валидацию логинов после конвертации в client_id
@@;

DEFINE ACTION $archive_search_queries() AS
$YT_SRC_INFO = $YT_BASE_PATH || '/' || $YT_TASK_FOLDER || '/' || $YT_REPORT_TYPE || '.info'; 
$YT_SRC_DATA = $YT_BASE_PATH || '/' || $YT_TASK_FOLDER || '/' || $YT_REPORT_TYPE || '.data'; 
$INFO = (
    SELECT SOME(TableRow())
    FROM $YT_SRC_INFO
);
$YT_DST_DATA = $YT_BASE_PATH || '/' || $YT_ARCHIVE_FOLDER || '/' || EvaluateExpr($INFO.DateFinish);

PRAGMA yt.PublishedErasureCodec = "lrc_12_2_2";
PRAGMA yt.NightlyCompress = "true";
INSERT INTO $YT_DST_DATA WITH TRUNCATE 
SELECT *
FROM $YT_SRC_DATA
; COMMIT;
PRAGMA yt.PublishedErasureCodec = default;
PRAGMA yt.NightlyCompress = default;
END DEFINE;

EVALUATE IF $MAKE_ARCHIVE DO $archive_search_queries();
COMMIT;


DEFINE ACTION $ensure_param_is_date($param_name, $date_str) AS
DISCARD SELECT Ensure(
  True,
  Cast($date_str AS Date) IS NOT NULL,
  "Неверный параметр " || $param_name || ': ' || $date_str
);
END DEFINE;

DEFINE ACTION $ensure_period($date_begin, $date_finish) AS
DISCARD SELECT Ensure(
  True,
    -- ограничение на максимальный период для отчета
  ($date_finish - $date_begin <= Interval('P90D')) AND ($date_finish - $date_begin >= Interval('P0D')),
  "Неверный период"
);
END DEFINE;

DO $ensure_param_is_date("$DATE_BEGIN", $DATE_BEGIN);
DO $ensure_param_is_date("$DATE_FINISH", $DATE_FINISH);
DO $ensure_period(Cast($DATE_BEGIN AS Date), Cast($DATE_FINISH AS Date));

$normalize_login = ($x) -> {
    RETURN String::ReplaceAll(String::ToLower($x), ".", "-");
};

-- Для обсчета по всем логинам
$HAS_LOGINS_WL = EvaluateExpr(ListLength($LOGINS) > 0);
$LOGINS = EvaluateExpr(ListMap($LOGINS, $normalize_login));

$CLIENT_IDS = (
    SELECT ClientID, String::JoinFromList(AGGREGATE_LIST(login), ",") AS login
    FROM `//home/direct/db/users`
    WHERE
      IF(
        $HAS_LOGINS_WL,
        $normalize_login(login) IN COMPACT $LOGINS,
        True
      )
    GROUP BY ClientID
);
-- используется для построения справочников по всем баннерам и фразам, в т.ч. архивным
$ORDER_ID_ALL = (
SELECT
  cid
, OrderID
, login
, name
, type
, currency
, archived
, statusActive
, o.ClientID AS ClientID
FROM `//home/direct/db/campaigns` AS c
INNER JOIN $CLIENT_IDS AS o
  USING(ClientID)
WHERE OrderID > 0
);

-- используется для отбора только актуальных событий из chevent-log'а
$HAS_CID_WL = EvaluateExpr(ListLength($CAMPAIGN_IDS_WL) > 0);
$ORDER_ID = (
SELECT cid, OrderID, login, name, type, currency, ClientID
FROM $ORDER_ID_ALL AS c
WHERE 
      type = "text"
  AND archived = "No"
  AND IF($HAS_CID_WL, cid IN COMPACT $CAMPAIGN_IDS_WL, True)
);
-- используется для построения справочника с сайтлинками
$CLID = (
SELECT DISTINCT ClientID
FROM $CLIENT_IDS
);
-- для отбора актуальных событий в @events
$ORD = (
SELECT DISTINCT OrderID 
FROM $ORDER_ID
);

-- для построения справочников по всем баннерам и фразам, в т.ч. архивным
$CID_ALL = (
SELECT DISTINCT cid AS CampaignID 
FROM $ORDER_ID_ALL
);

$CID_MIN_MAX = (SELECT MIN(CampaignID) AS min_cid, MAX(CampaignID) AS max_cid FROM $CID_ALL);
$CID_MIN = (SELECT min_cid FROM $CID_MIN_MAX);
$CID_MAX = (SELECT max_cid FROM $CID_MIN_MAX);

PRAGMA yt.DataSizePerJob = "10G";

INSERT INTO @bids WITH TRUNCATE 
SELECT cid, pid, id, phrase, PhraseID
FROM `home/direct/db/bids`
WHERE cid >= $CID_MIN 
  AND cid <= $CID_MAX
  AND IF($HAS_LOGINS_WL, cid IN COMPACT $CID_ALL, cid IN $CID_ALL)
  AND PhraseID > 0
UNION ALL
SELECT cid, pid, id, phrase, PhraseID
FROM `home/direct/db/bids_arc`
WHERE cid >= $CID_MIN 
  AND cid <= $CID_MAX
  AND IF($HAS_LOGINS_WL, cid IN COMPACT $CID_ALL, cid IN $CID_ALL)
  AND PhraseID > 0
;

INSERT INTO @sitelinks WITH TRUNCATE 
SELECT sitelinks_set_id, String::JoinFromList(AGGREGATE_LIST(title), ", ") AS SLText
FROM `home/direct/db/sitelinks_sets`
WHERE IF($HAS_LOGINS_WL, ClientID IN COMPACT $CLID, ClientID IN $CLID)
GROUP BY sitelinks_set_id
;

INSERT INTO @banners_prefilter WITH TRUNCATE
SELECT 
  BannerID
, image_BannerID
, title
, body
, sitelinks_set_id
, bid
, pid
, href
FROM `home/direct/db/banners`
WHERE cid >= $CID_MIN 
  AND cid <= $CID_MAX
  AND IF($HAS_LOGINS_WL, cid IN COMPACT $CID_ALL, cid IN $CID_ALL)
  AND (BannerID > 0 OR image_BannerID > 0)
; COMMIT;

INSERT INTO @banners WITH TRUNCATE
SELECT 
  BannerID
, title
, body
, sitelinks_set_id
, bid
, pid
, href
FROM @banners_prefilter
WHERE BannerID > 0
UNION ALL
SELECT 
  image_BannerID AS BannerID
, title
, body
, sitelinks_set_id
, bid
, pid
, href
FROM @banners_prefilter
WHERE image_BannerID > 0
; 

-- https://yql.yandex-team.ru/Operations/W-wv3WUObwz4ssymD3YZXAuetOvYXmsgx4Ho_Rs3lgc=
$PAGE_ID = "//home/vipplanners/dict/ta/search_pageid_top50";
$_PAGE_ID_ALL = (SELECT AGGREGATE_LIST(pageid) AS pageid FROM $PAGE_ID);
-- в кортеж, чтобы использовалась оптимизация с выбором диапазонов
$PAGE_ID_ALL = EvaluateCode(ListCode(ListMap(Unwrap($_PAGE_ID_ALL), ($i)->{return ReprCode($i)})));

INSERT INTO @events WITH TRUNCATE 
SELECT 
  searchquery
, contexttype
, positionctrcorrection
, pageid
, eventcost
, IF(eventcost > 0, eventcostcur, 0) AS eventcostcur
, countertype
, orderid
, bannerid
-- phraseid = 1 служебный для ДРФ, фактическая привязка в targetphraseid
-- если phraseid = targetphraseid = 1, привязка по баннеру
, IF(phraseid == 1, targetphraseid, phraseid) AS phraseid
, relevance
, maxpositionctrcorrection
FROM RANGE(`//cooked_logs/bs-chevent-cooked-log/1d`, $DATE_BEGIN, $DATE_FINISH)
WHERE pageid IN COMPACT $PAGE_ID_ALL
  AND contexttype IN (1, 3, 5)  -- https://wiki.yandex-team.ru/bannernajakrutilka/ContextType/
  AND selecttype = 5  -- 100% поиск (https://wiki.yandex-team.ru/users/ilariia/selecttypes/)
  AND fraudbits = 0
  AND (countertype == 1 OR countertype == 2)  -- Показы или клики
  AND searchquery > ''
  AND phraseid > 0
  AND IF($HAS_LOGINS_WL, orderid IN COMPACT $ORD, orderid IN $ORD)
; COMMIT;

PRAGMA yt.DataSizePerMapJob = default;

$CAMPAIGN_TYPE = AsDict(
    ("text", "TEXT_CAMPAIGN")
);
-- https://wiki.yandex-team.ru/bannernajakrutilka/ContextType/
-- https://tech.yandex.ru/direct/doc/reports/report-format-docpage/
-- https://a.yandex-team.ru/arc/trunk/arcadia/yabs/interface/yabs-export-scripts-fast/Yabs/Stat/MasterReport/QueryCommon.pm?blame=true&rev=3991156#L287-299
-- https://st.yandex-team.ru/BSDEV-64394
$MATCH_TYPE = AsDict(
    (1, "NONE"),
    (3, "RELATED_KEYWORD"),
    (5, "SYNONYM")
);

INSERT INTO @preagg WITH TRUNCATE 
SELECT 
  Query
, MatchType
, SOME(orderid) AS OrderID
, BSBannerID
, PhraseID
, PageID
, SUM(IF(countertype == 1, 1, 0)) AS Shows
  -- https://st.yandex-team.ru/BSSERVER-5051
, SUM(IF(countertype == 1, positionctrcorrection, 0)) AS eShows  -- https://wiki.yandex-team.ru/users/maxlog/work/new-auction/e-ctr/
, SUM(IF(countertype == 1, maxpositionctrcorrection, 0)) AS MaxX
, SUM(IF(countertype == 2, 1, 0)) AS Clicks
, SUM(eventcost) / 1e6 AS CostUnit  -- фишки
, SUM(eventcostcur) / 1e6 AS Cost  -- валюта логина
, SUM(IF(countertype == 1, relevance, 0)) AS ShowsSumRel
, SUM(IF(countertype == 2, relevance, 0)) AS ClicksSumRel
FROM @events
GROUP BY
  searchquery AS Query
, CASE contexttype 
    WHEN 1 THEN $MATCH_TYPE[1]
    WHEN 3 THEN $MATCH_TYPE[3]
    WHEN 5 THEN $MATCH_TYPE[5]
    ELSE "UNKNOWN"
  END AS MatchType -- источник условия таргетирования
, bannerid AS BSBannerID
, phraseid AS PhraseID
, pageid AS PageID
; COMMIT;

INSERT INTO @data WITH TRUNCATE 
SELECT
  Query
, MatchType
, o.login AS Login
, o.ClientID AS ClientID
, e.OrderID AS OrderID
, o.cid AS CampaignID
, o.name AS CampaignName
, $CAMPAIGN_TYPE[o.type]??"UNKNOWN" AS CampaignType
, o.currency AS Currency
, e.BSBannerID AS BSBannerID
, e.PhraseID AS PhraseID
, p.domain_full AS page_domain_full
, IF(p.is_yandex_group, "Яндекс", p.domain_full) AS AdvSite
, Shows
, eShows
, MaxX
, Clicks
, CostUnit
, ShowsSumRel
, ClicksSumRel
-- Ratio из https://yt.yandex-team.ru/hahn/navigation?path=//home/yabs/dict/Currency
, CASE o.currency
    WHEN "UAH" THEN Cost * 10
    WHEN "RUB" THEN Cost * 100
    WHEN "BYN" THEN Cost * 100
    WHEN "KZT" THEN Cost * 100
    WHEN "BYR" THEN Cost * 1000000
    ELSE Cost
  END AS Cost
FROM @preagg AS e
INNER JOIN $ORDER_ID AS o
  ON e.OrderID = o.OrderID
INNER JOIN $PAGE_ID AS p
  ON e.PageID = p.pageid
; COMMIT;

INSERT INTO @preagg_2 WITH TRUNCATE
SELECT 
  Query
, MatchType
, SOME(Login) AS Login
, SOME(ClientID) AS ClientID
, SOME(OrderID) AS OrderID
, SOME(CampaignID) AS CampaignID
, SOME(CampaignName) AS CampaignName
, SOME(CampaignType) AS CampaignType
, SOME(Currency) AS Currency
, BSBannerID
, PhraseID
, AdvSite
, SUM(Shows) AS Shows
, SUM(eShows) AS eShows
, SUM(MaxX) AS MaxX
, SUM(Clicks) AS Clicks
, SUM(CostUnit) AS CostUnit
, SUM(Cost) AS Cost
, SUM(ShowsSumRel) AS ShowsSumRel
, SUM(ClicksSumRel) AS ClicksSumRel
FROM @data
GROUP BY
  Query
, MatchType
, BSBannerID
, PhraseID
, AdvSite
; COMMIT;

-- Все исходные данные, далее только дополнительные вычисления
INSERT INTO @data_final_j1 WITH TRUNCATE
SELECT
  Query
, MatchType
, Login
, ClientID
, OrderID
, CampaignID
, CampaignName
, CampaignType
, Currency
, b.pid AS GroupID
, b.bid AS DirectBannerID
, BSBannerID
, PhraseID
, AdvSite
, Shows
, eShows
, MaxX
, Clicks
, CostUnit
, Cost
, ShowsSumRel
, ClicksSumRel
, b.title AS TitleDB
, b.body AS BodyDB
, b.href AS Href
, b.sitelinks_set_id AS sitelinks_set_id
FROM @preagg_2 AS j0
LEFT JOIN @banners AS b
  ON j0.BSBannerID = b.BannerID
; COMMIT;

INSERT INTO @data_final_j2 WITH TRUNCATE
SELECT
  Query
, MatchType
, Login
, ClientID
, OrderID
, CampaignID
, CampaignName
, CampaignType
, Currency
, GroupID
, DirectBannerID
, BSBannerID
, PhraseID
, AdvSite
, Shows
, eShows
, MaxX
, Clicks
, CostUnit
, Cost
, ShowsSumRel
, ClicksSumRel
, TitleDB
, BodyDB
, Href
, SLText
FROM @data_final_j1 AS j1
LEFT JOIN @sitelinks AS sl
  ON j1.sitelinks_set_id = sl.sitelinks_set_id
WHERE j1.sitelinks_set_id > 0
UNION ALL
SELECT
  Query
, MatchType
, Login
, ClientID
, OrderID
, CampaignID
, CampaignName
, CampaignType
, Currency
, GroupID
, DirectBannerID
, BSBannerID
, PhraseID
, AdvSite
, Shows
, eShows
, MaxX
, Clicks
, CostUnit
, Cost
, ShowsSumRel
, ClicksSumRel
, TitleDB
, BodyDB
, Href
, CAST(NULL AS String) AS SLText
FROM @data_final_j1
WHERE sitelinks_set_id IS NULL
; COMMIT;

INSERT INTO @data_final_j3 WITH TRUNCATE
SELECT
  Query
, MatchType
, Login
, ClientID
, OrderID
, CampaignID
, CampaignName
, CampaignType
, Currency
, GroupID
, DirectBannerID
, BSBannerID
, j2.PhraseID AS PhraseID
, bid.id AS DirectPhraseID
, bid.phrase AS PhraseText
, AdvSite
, Shows
, eShows
, MaxX
, Clicks
, CostUnit
, Cost
, ShowsSumRel
, ClicksSumRel
, TitleDB
, BodyDB
, Href
, SLText
FROM @data_final_j2 AS j2
LEFT JOIN @bids AS bid
  ON j2.PhraseID = bid.PhraseID
    AND j2.GroupID = bid.pid
; COMMIT;

PRAGMA yt.MaxJobCount = "65536";
PRAGMA yt.DefaultMemoryLimit = "8G";
PRAGMA yt.DataSizePerJob = "50M";

PRAGMA yt.PoolTrees = "physical";
PRAGMA yt.TentativePoolTrees = "cloud";
-- собрано отсюда https://a.yandex-team.ru/arc/trunk/arcadia/junk/n-bar/word_utils_udf
PRAGMA File('word_utils.so', 'yt://hahn/home/vipplanners/ta_report/udf/libword_utils_udf.so');
PRAGMA udf('word_utils.so');
PRAGMA File('transform_template.py', 'yt://hahn/home/vipplanners/ta_report/udf/transform_template.py');

$_format = ArcPython2::try_format_template(
        Callable<(Utf8?, Utf8?, Uint32?) -> Utf8?>,
        FileContent("transform_template.py")
);
$render_title = ($template, $phrase) -> {
  RETURN $_format(
    CAST($template AS Utf8),
    CAST($phrase AS Utf8),
    35  -- продуктовое ограничение на размер строки
  )
};
$render_body = ($template, $phrase) -> {
  RETURN $_format(
    CAST($template AS Utf8),
    CAST($phrase AS Utf8),
    81  -- продуктовое ограничение на размер строки
  )
};

PRAGMA File("word_helper.py", "yt://hahn/home/vipplanners/ta_report/udf/word_helper.py");

$criteria_num_words = CustomPython::criteria_num_words(
    Callable<(Utf8)->Uint8?>,
    FileContent("word_helper.py")
);
$query_num_words = CustomPython::query_num_words(
    Callable<(Utf8, Utf8)->Uint8?>,
    FileContent("word_helper.py")
);
$get_hl_nhl = CustomPython::get_hl_nhl(
-- (criteria, query, snippet)
    Callable<(Utf8, Utf8, Utf8) -> 
        Struct<
          criteria:Utf8,
          query:Utf8,
          snippet:Utf8,
          hl:List<Utf8>?,
          nhl:List<Utf8>?
        >>,
    FileContent("word_helper.py")
);
-- https://wiki.yandex-team.ru/users/insight/ads-dssms/
-- исходня модель из https://proxy.sandbox.yandex-team.ru/208065656
-- свежая модель:
--   вики: из https://wiki.yandex-team.ru/users/jruziev/dssm-search-models/#reguljarnyjjprocess
--   тикете: https://st.yandex-team.ru/BSDEV-73098
-- сконвертирова для применения как ./arcadia/ml/dssm/dssm/dssm3 convert4apply -d <fname> -o model.apply
$SB_OLD_MODEL_PATH = "yt://hahn/home/vipplanners/ta_report/udf/model.apply";
PRAGMA File('dssm_model.apply', $SB_OLD_MODEL_PATH);

-- https://st.yandex-team.ru/BSDEV-73098#5ca5d56d977527001ff4f719
-- https://wiki.yandex-team.ru/users/jruziev/dssm-search-models/#reguljarnyjjprocess
$SB_TASK_INFO_PATH = @@https://sandbox.yandex-team.ru/api/v1.0/resource?resource_type=ML_ENGINE_DUMP&state=READY&order=-created&limit=1&attrs={"engine":%20"dssm",%20"subtask_id":%20"assessor.appl",%20"task_id":%20"PROD_search_assessor_dssm"}@@;
PRAGMA File("task_info.json", $SB_TASK_INFO_PATH);
$YT_NEW_MODEL_PATH = (SELECT Yson::ConvertToString(Yson::ParseJson(FileContent("task_info.json"))["items"][0]["http"]["proxy"]) || "/assessor.appl");
PRAGMA File('new_dssm_model.apply', $YT_NEW_MODEL_PATH);

$ads_model_0 = Dssm::LoadModel(FilePath("dssm_model.apply"));
$ads_model_1 = Dssm::LoadModel(FilePath("new_dssm_model.apply"));
$replace_url_tail = Re2::Replace("([?%#&=].*)$");
$strip_url = ($url) -> {
    RETURN $replace_url_tail($url, '');
};
-- адаптация https://github.yandex-team.ru/custom-solutions/constructor/blob/4071a2a7e7d956be2069da50a2268ac49d9e2753/interester/highlighter/reports/search_queries.py#L297
$apply_ads_model_0 = ($query, $title, $body, $href) -> {
  RETURN Dssm::Apply(
    $ads_model_0,
    AsStruct(
      $query??'' AS query,
      '' AS region,
      $title??'' AS title,
      $body??'' AS snippet,
      $strip_url($href)??'' AS stripped_goodurl,
      Url::GetHost($strip_url($href)??'')??'' AS domain,
-- в исходной версии применялся ads/quality/apc/apc_tools fast_uta для stripped_url 
      '' AS goodurl_uta
    ),
  "joint_output"
  )[0]; -- в списке всегда одно значение
};

$apply_ads_model_1 = ($query, $title, $body, $href) -> {
  RETURN Dssm::Apply(
    $ads_model_1,
    AsStruct(
      $query??'' AS query,
      '' AS region_id,
      $title??'' AS title,
      $body??'' AS snippet,
      $strip_url($href)??'' AS stripped_goodurl,
      '' AS domain_id,
-- в исходной версии применялся ads/quality/apc/apc_tools fast_uta для stripped_url 
      '' AS goodurl_uta
    ),
  "joint_output"
  )[0]; -- в списке всегда одно значение
};


$_GET_HL_NHL = ($row) -> {
  RETURN $get_hl_nhl(
      CAST($row.PhraseText AS Utf8)??Utf8(""),
      CAST($row.Query AS Utf8)??Utf8(""),
      CAST($render_title($row.TitleDB, $row.PhraseText) AS Utf8)??Utf8("")
      || Utf8(" ")
      || CAST($render_body($row.BodyDB, $row.PhraseText) AS Utf8)??Utf8("")
  )
};

$YT_FULL_PATH = $YT_BASE_PATH || '/' || $YT_TASK_FOLDER || '/' || $YT_REPORT_TYPE;
$YT_FULL_PATH_RESULT = $YT_FULL_PATH || ".data";

$Title = ($row) -> {
  RETURN $render_title($row.TitleDB, $row.PhraseText)??''
};
$Body = ($row) -> {
  RETURN $render_body($row.BodyDB, $row.PhraseText)??''
};

$DSSMAds0 = ($row) -> {
    RETURN $apply_ads_model_0(
    $row.Query,
    $Title($row),
    $Body($row),
    $row.Href
  )
};
$DSSMAds1 = ($row) -> {
    RETURN $apply_ads_model_1(
    $row.Query,
    $Title($row),
    $Body($row),
    $row.Href
  )
};

$calc_prob = ($x) -> {
  RETURN Math::Exp($x) / (1. + Math::Exp($x))
};

INSERT INTO @data_final_j4 WITH TRUNCATE
SELECT
  Query
, MatchType
, Login
, ClientID
, OrderID
, CampaignID
, CampaignName
, CampaignType
, Currency
, GroupID
, DirectBannerID
, BSBannerID
, DirectPhraseID 
, PhraseText
, AdvSite
, Shows??0 AS Shows
, eShows??0 AS eShows
, MaxX??0 AS MaxX
, Clicks??0 AS Clicks
, CostUnit??0 AS CostUnit
, Cost??0 AS Cost
, ShowsSumRel??0 AS ShowsSumRel
, IF(Shows > 0, (ShowsSumRel * 1.) / Shows, 0) AS ShowsRelevance
, ClicksSumRel??0 AS ClicksSumRel
, IF(Clicks > 0, (ClicksSumRel * 1.) / Clicks, 0) AS ClicksRelevance
, $Title(TableRow()) AS Title
, $Body(TableRow()) AS Body
, TitleDB
, BodyDB
, Href
, SLText
, $criteria_num_words(
    CAST(PhraseText AS Utf8)??Utf8("")
  )??0 AS PhraseWordsCnt
, $query_num_words(
    CAST(PhraseText AS Utf8)??Utf8(""),
    CAST(Query AS Utf8)??Utf8("")
  )??0 AS QueryWordsCnt
, $calc_prob($DSSMAds0(TableRow())) AS DSSMScoreOld
, $calc_prob($DSSMAds1(TableRow())) AS DSSMScore
, ListLength($_GET_HL_NHL(TableRow()).hl ?? ListCreate(Utf8)) AS QueryHighlighTitleBody
, String::JoinFromList(ListSortAsc($_GET_HL_NHL(TableRow()).nhl??ListCreate(Utf8)), Utf8(", ")) AS QueryNonBoldedWordsTitleBody
FROM @data_final_j3
; COMMIT;

PRAGMA yt.MaxJobCount = default;
PRAGMA yt.DefaultMemoryLimit = default;
PRAGMA yt.DataSizePerJob = default;
PRAGMA yt.TentativePoolTrees = default;
PRAGMA ClassicDivision = "false";

INSERT INTO $YT_FULL_PATH_RESULT WITH TRUNCATE
SELECT
  Query
, MatchType
, Login
, ClientID
, OrderID
, CampaignID
, CampaignName
, CampaignType
, Currency
, GroupID
, DirectBannerID
, BSBannerID
, DirectPhraseID 
, PhraseText
, AdvSite
, Shows
, eShows
, MaxX
, Clicks
, CostUnit
, Cost
, ShowsSumRel
, ShowsRelevance
, ClicksSumRel
, ClicksRelevance
, Title
, Body
, TitleDB
, BodyDB
, Href
, SLText
, PhraseWordsCnt
, QueryWordsCnt
, DSSMScoreOld
-- PLANNERSDEVTA-163
, DSSMScore + DSSMScore * (1 - DSSMScore) AS DSSMScore
, QueryHighlighTitleBody
, QueryNonBoldedWordsTitleBody
FROM @data_final_j4
ORDER BY ClientID, CampaignID
; COMMIT;

$RESULT_LOG_LINES = (SELECT COUNT(*) FROM $YT_FULL_PATH_RESULT);
$YT_FULL_PATH_ERRORS = $YT_FULL_PATH || ".errors";

INSERT INTO $YT_FULL_PATH_ERRORS WITH TRUNCATE 
SELECT
  AsTuple(
    COUNT_IF(Shows > 0),
    COUNT_IF(Shows > 0) / $RESULT_LOG_LINES
  ) AS Shows
, AsTuple(
    COUNT_IF(Query > ''),
    COUNT_IF(Query > '') / $RESULT_LOG_LINES
  ) AS Query
, AsTuple(
    COUNT_IF(PhraseText > ''),
    COUNT_IF(PhraseText > '') / $RESULT_LOG_LINES
  ) AS PhraseText
, AsTuple(
    COUNT_IF(Title > ''),
    COUNT_IF(Title > '') / $RESULT_LOG_LINES
  ) AS Title
, AsTuple(
    COUNT_IF(Body > ''),
    COUNT_IF(Body > '') / $RESULT_LOG_LINES
  ) AS Body
, AsTuple(
    COUNT_IF(Href > ''),
    COUNT_IF(Href > '') / $RESULT_LOG_LINES
  ) AS Href
, AsTuple(
    COUNT_IF(PhraseWordsCnt > 0),
    COUNT_IF(PhraseWordsCnt > 0) / $RESULT_LOG_LINES
  ) AS PhraseWordsCnt
, AsTuple(
  COUNT_IF(QueryHighlighTitleBody > 0 OR QueryNonBoldedWordsTitleBody > ''),
  COUNT_IF(QueryHighlighTitleBody > 0 OR QueryNonBoldedWordsTitleBody > '') / $RESULT_LOG_LINES
  ) AS HasHLNHL
, AsTuple(
    COUNT_IF(QueryWordsCnt > 0),
    COUNT_IF(QueryWordsCnt > 0) / $RESULT_LOG_LINES
  ) AS QueryWordsCnt
, AsTuple(
    COUNT_IF(DSSMScoreOld IS NOT NULL),
    COUNT_IF(DSSMScoreOld IS NOT NULL) / $RESULT_LOG_LINES
  ) AS DSSMScoreOld
, AsTuple(
    COUNT_IF(DSSMScore IS NOT NULL),
    COUNT_IF(DSSMScore IS NOT NULL) / $RESULT_LOG_LINES
  ) AS DSSMScore
FROM $YT_FULL_PATH_RESULT
; COMMIT;

PRAGMA ClassicDivision = "true";

SELECT *
FROM $YT_FULL_PATH_ERRORS
INTO RESULT `Errors`
;
$YT_FULL_PATH_INFO = $YT_FULL_PATH || ".info";
INSERT INTO $YT_FULL_PATH_INFO WITH TRUNCATE 
SELECT 
  $QUERY_VERSION AS QueryVersion
, $DATE_BEGIN AS DateBegin
, $DATE_FINISH AS DateFinish
; COMMIT;
SELECT * FROM $YT_FULL_PATH_INFO
INTO RESULT `Info`
;
