/* OPERANALYTICS-414: Доработки таблиц с почтой для рассылок

1. Все адреса объединяются в таблице all_emails c полями ClientID,
   email, dt (дата добавления адреса), Source (источник адреса):
- адрес почтового обращения в Space (home/crm/v_ticket)
- адрес из телефонного апсейла (home/crm/v_call)
- адрес из direct.users для новостей и рекомендаций (поле recommendations_email)
- адрес из direct.users для уведомлений (автоматический) (поле email)
- адрес из direct.users логинный (добавляется @yandex.ru и всевозможные вариации)
- адрес из persons-clients (home/balance/prod/bo/t_person)
2. Сбор информации по рассылкам
- факт отправления рассылки //home/logfeller/logs/sendr-delivery-log/1d за всю историю
- действия пользователя с рассылкой //statbox/sendr-click-log за всю историю
3. Мэтчинг всех адресов и информации по рассылкам в одну таблицу
   //home/comdep-analytics/nkapelko/rassylki/all
- Source - источник адреса
- dt_sent - дата рассылки
- campaign_id - id рассылки
- status_sent = 1 рассылка отправлялась
- status_open = 1 рассылка была открыта с этого адреса
- status_unsub = 1 адрес отписался от рассылки
4. Из всех адресов далее выбирается самый актуальный адрес - это
   адрес, на который пришла последняя открытая рассылка. Если таких адресов
   несколько, то адрес выбирается согласно ранжированию по приоритету источников.
   Если открытий рассылок не было, то адрес выбирается также согласно
   ранжированию источников. Общую логику удобнее посмотреть на схеме в
   файле https://wiki.yandex-team.ru/users/nkapelko/emails/.files/logikamonitoringaiapdeitajel.adresov-finalsbillingom.xlsx
*/
PRAGMA DisableAnsiInForEmptyOrNullableItemsCollections;
PRAGMA yt.ParallelOperationsLimit = "7";  -- дефолтный лимит на пользователя 10
PRAGMA yt.DefaultOperationWeight = "0.5"; -- дефолт 1.0
PRAGMA yt.PoolTrees = "physical";
PRAGMA yt.TentativePoolTrees = "cloud";

PRAGMA File('libcrypta_identifier_udf.so', 'yt://hahn/home/crypta/public/udfs/stable/libcrypta_identifier_udf.so');
PRAGMA Udf('libcrypta_identifier_udf.so');

USE hahn;

$PATH_ROOT = "//home/vipplanners/rassylki/v2";


$match = Re2::Match(@@.*yandex-team\..*@@);
$match_ya1 = Re2::Match(@@.*support\.yandex\..*@@);
$match_ya2 = Re2::Match(@@.*money\.yandex\..*@@);
$match_ya3 = Re2::Match(@@.*direct\.yandex\..*@@);
$match_ya4 = Re2::Match(@@.*balance\.yandex\..*@@);
$match_ya5 = Re2::Match(@@.*adv\.yandex\..*@@);
$match_ya6 = Re2::Match(@@.*destek\.yandex\..*@@);
$match_ya7 = Re2::Match(@@.*partner\.yandex\..*@@);
$match_ya8 = Re2::Match(@@.*yndx.*@@);  
$match_ya9 = Re2::Match(@@.*localhost\.balance.*@@);
$match_ya10 = Re2::Match(@@.*taxi\.yandex\..*@@);
$match_ya11 = Re2::Match(@@.*host\.yandex\..*@@);
$match_ya12 = Re2::Match(@@.*zen\.yandex\..*@@);
$match_ya13 = Re2::Match(@@.*delivery\.yandex\..*@@);

$is_internal_email = ($x) -> {
  RETURN $match($x)
      OR $match_ya1($x)
      OR $match_ya2($x)
      OR $match_ya3($x)
      OR $match_ya4($x)
      OR $match_ya5($x)
      OR $match_ya6($x)
      OR $match_ya7($x)
      OR $match_ya8($x)
      OR $match_ya9($x)
      OR $match_ya10($x)
      OR $match_ya11($x)
      OR $match_ya12($x)
      OR $match_ya13($x);
};

$normalize_email_safe = ($x) -> {
  -- без замены значимых символов и нормализации домена
  RETURN String::RemoveAll(String::Strip(String::ToLower($x)), " ,;\t\r\n");
};

$split_to_emails = ($emails) -> {
  RETURN ListFilter(ListMap(String::SplitToList($emails, ",; ", False AS DelimeterString), $normalize_email_safe), ($x) -> ($x > ''))
};

/* 1. Сбор всех источников */
-- адрес почтового обращения в Space
$PATH_SPACE = $PATH_ROOT || "/sources/email_space";
INSERT INTO $PATH_SPACE WITH TRUNCATE
SELECT
  client_id AS ClientID
, eml AS email
, ticket_dt AS dt
, "space" AS SOURCE
FROM `//home/crm/v_ticket`
FLATTEN LIST BY ($split_to_emails(ticket_from_email) AS eml)
WHERE client_id IS NOT NULL
  AND eml IS NOT NULL
  AND NOT $is_internal_email(eml)
;

-- адрес из телефонного апсейла
$PATH_UPSALE = $PATH_ROOT || "/sources/email_upsale";
INSERT INTO $PATH_UPSALE WITH TRUNCATE
SELECT
  client_id AS ClientID
, eml AS email
, call_dt AS dt
, "upsale" AS SOURCE
FROM `//home/crm/v_call`
FLATTEN LIST BY ($split_to_emails(email) AS eml)
WHERE client_id IS NOT NULL
  AND eml IS NOT NULL
  AND mail_sent = 1
  AND NOT $is_internal_email(eml)
;

-- адрес из direct.users для новостей и рекомендаций
$PATH_RECOM = $PATH_ROOT || "/sources/email_recommendations";
INSERT INTO $PATH_RECOM WITH TRUNCATE
SELECT
  ClientID
, eml AS email
, create_time AS dt
,  "recommendations" AS SOURCE
FROM `//home/direct/db/users`
FLATTEN LIST BY ($split_to_emails(recommendations_email) AS eml)
WHERE eml IS NOT NULL
  AND ClientID IS NOT NULL
  AND NOT $is_internal_email(eml)
;

-- адрес из direct.users для уведомлений (автоматический)
$PATH_NOTIF = $PATH_ROOT || "/sources/email_notifications";
INSERT INTO $PATH_NOTIF WITH TRUNCATE
SELECT
  ClientID
, email
, dt
, SOURCE
FROM (
  SELECT
    ClientID
  , email
  , dt
  , SOURCE
  FROM (
    SELECT
      ClientID
    , CASE
        WHEN email = (CAST($normalize_email_safe(login) AS String) || '@yandex.ru') THEN Nothing(String?)
        ELSE $normalize_email_safe(email)
      END AS email
    , create_time AS dt
    ,  "notifications" AS SOURCE
    FROM `//home/direct/db/users`
    WHERE ClientID IS NOT NULL
  )
  UNION ALL
  SELECT
    ClientID
  , email
  , dt
  , SOURCE
  FROM (
    SELECT
      ClientID
    , email
    , max(dt) AS dt
    , SOURCE
    FROM  (
      SELECT
        a.ClientID AS ClientID
      , CASE
          WHEN email = (CAST($normalize_email_safe(login) AS String) || '@yandex.ru') THEN Nothing(String?)
          ELSE $normalize_email_safe(email)
        END AS email
      , create_time AS dt
      ,  "notifications" AS SOURCE
      FROM `//home/direct/db/campaigns` AS a
      LEFT JOIN (
        SELECT
          ClientID
        , login
        FROM `//home/direct/db/users`
        WHERE ClientID IS NOT NULL
        ) AS b
        ON a.ClientID = b.ClientID
      WHERE a.ClientID IS NOT NULL
    )
    GROUP BY ClientID, email, SOURCE
  )
)
WHERE email IS NOT NULL
  AND NOT $is_internal_email(email)
;

-- адрес из direct.users логинный - @yandex.ru
$replace = Re2::Replace("-");
$PATH_LOGIN = $PATH_ROOT || "/sources/email_login";
INSERT INTO $PATH_LOGIN WITH TRUNCATE
SELECT
  ClientID
, logins || domains AS email
, dt
, "login" AS SOURCE
FROM (
  SELECT
    ClientID
  , DictKeys(AsSet($normalize_email_safe(login), $replace($normalize_email_safe(login), '.'))) AS logins
  , AsList('@yandex.ru', '@yandex.com', '@yandex.by', '@yandex.kz', '@ya.ru', '@narod.ru') AS domains
  , create_time AS dt
  FROM `//home/direct/db/users`
  WHERE ClientID IS NOT NULL
    AND login > ''
)
FLATTEN LIST BY (logins, domains)
;

-- адрес из t_client Баланса
$PATH_PERS = $PATH_ROOT || "/sources/email_persons";
INSERT INTO $PATH_PERS WITH TRUNCATE
SELECT
  id AS ClientID
, eml AS email
, 'persons-clients email' AS SOURCE
FROM `//home/balance/prod/bo/t_client`
FLATTEN LIST BY ($split_to_emails(email) AS eml)
WHERE id IS NOT NULL
  AND eml IS NOT NULL
  AND NOT $is_internal_email(eml)
;

-- адрес из t_person
$PATH_PERS_EMAIL = $PATH_ROOT || "/sources/email_person_email";
INSERT INTO $PATH_PERS_EMAIL WITH TRUNCATE
SELECT
  client_id AS ClientID
, eml AS email
, 'persons-clients person_email' AS SOURCE
FROM `//home/balance/prod/bo/t_person`
FLATTEN LIST BY ($split_to_emails(email) AS eml)
WHERE client_id IS NOT NULL
  AND eml IS NOT NULL
  AND NOT $is_internal_email(eml)
;

/* 2. Сбор информации по рассылкам */
-- рассылки: отправленые, открыты, отписки
$PATH_SENT = $PATH_ROOT || "/sendr/sent";
INSERT INTO $PATH_SENT WITH TRUNCATE
SELECT
  campaign_id
, iso_eventtime
, $normalize_email_safe(recepient) AS recepient
FROM RANGE(`//home/logfeller/logs/sendr-delivery-log/1d`)
WHERE status = "0"
  AND account = 'direct'
  AND NOT $is_internal_email(recepient)
;

$PATH_OPENED = $PATH_ROOT || "/sendr/opened";
INSERT INTO $PATH_OPENED WITH TRUNCATE
SELECT
  campaign_id
, email
, iso_eventtime
, event
FROM (
  SELECT
    campaign_id
  , WeakField(iso_eventtime, String) AS iso_eventtime
  , $normalize_email_safe(email) AS email
  , event
  FROM RANGE(`//statbox/sendr-click-log`)
  WHERE account = 'direct'
    AND event IN ('px', 'click')
     OR event = 'unsub'
)
GROUP BY
  campaign_id
, email
, iso_eventtime
, IF(event = 'unsub', 'unsubscribed', 'opened') AS event
;

/* 3. Мэтчинг всех адресов и информации по рассылкам в одну таблицу */
COMMIT;
-- сведение адресов
$replace_ya = Re2::Replace(@@ya\.ru@@);
$replace_yandex = Re2::Replace(@@yandex\.ru@@);

$match_ya = Re2::Match(@@.*\@ya\.ru@@);
$match_yandex = Re2::Match(@@.*\@yandex\.ru@@);

$agency_host_bl = (
  "elama.ru",
  "r-broker.ru",
  "click.ru",
  "aori.ru",
  "admitad.com",
  "tiu.ru",
  "megagroup.ru",
  "orgm.ru",
  "nominaltechno.ru",
  "ex-in.ru",
);

$is_agency_host = ($email) -> {
  $host = String::SplitToList(String::Strip($email), '@', false, true)[1];
  RETURN $host IN $agency_host_bl;
};

$PATH_ALL_EMAIL = $PATH_ROOT || "/union/all_emails";
INSERT INTO $PATH_ALL_EMAIL WITH TRUNCATE
SELECT
  ClientID
, email
, SOURCE AS Source
, MAX(cu.dt) AS dt
FROM (
    SELECT ClientID, email, dt, SOURCE
    FROM $PATH_LOGIN
      UNION ALL
    SELECT CAST(ClientID AS Int64) AS ClientID, email, dt, SOURCE
    FROM $PATH_SPACE
      UNION ALL
    SELECT CAST(ClientID AS Int64) AS ClientID, email, dt, SOURCE
    FROM $PATH_UPSALE
    WHERE CAST(ClientID AS Int64) > 0
      UNION ALL
    SELECT ClientID, email, dt, SOURCE
    FROM $PATH_NOTIF
      UNION ALL
    SELECT ClientID, $replace_ya(email, "yandex.ru") AS email, dt, SOURCE
    FROM $PATH_NOTIF
    WHERE $match_ya(email)
      UNION ALL
    SELECT ClientID, $replace_yandex(email, "ya.ru") AS email, dt, SOURCE
    FROM $PATH_NOTIF
    WHERE $match_yandex(email)
      UNION ALL
    SELECT ClientID, email, dt, SOURCE
    FROM $PATH_RECOM
      UNION ALL
    SELECT ClientID, $replace_ya(email, "yandex.ru") AS email, dt, SOURCE
    FROM $PATH_RECOM
    WHERE $match_ya(email)
      UNION ALL
    SELECT ClientID, $replace_yandex(email, "ya.ru") AS email, dt, SOURCE
    FROM $PATH_RECOM
    WHERE $match_yandex(email)
      UNION ALL
    SELECT ClientID, email, "0000-00-00T00:00:00" AS dt, SOURCE
    FROM $PATH_PERS
      UNION ALL
    SELECT ClientID, email, "0000-00-00T00:00:00" AS dt, SOURCE
    FROM $PATH_PERS_EMAIL
) AS cu
LEFT JOIN `//home/comdep-analytics/public/client_tiers/fact/latest_v2` AS crm
  ON cu.ClientID = crm.client_id
WHERE Identifiers::IsValidEmail(cu.email)
  AND (
       crm.client_id IS NULL
    OR crm.agency_id > 0
    OR  (
        NVL(crm.agency_id, 0) = 0  -- прямой клиент
    AND NOT $is_agency_host(cu.email)  -- неагентская почта
    )
  )
GROUP BY
  cu.ClientID AS ClientID
, cu.email AS email
, cu.SOURCE AS SOURCE
ORDER BY
  ClientID
, Source
;

-- сведение адресов и рассылок
COMMIT;  -- after all_email

$normalize_ya_email = ($a) -> {
  $replace = Re2::Replace("@yandex.by|@yandex.kz|@yandex.ua|@yandex.com|@ya.ru|@narod.ru");
  $replace2 = Re2::Replace(" | ");
  $first_email = ListTake((String::SplitToList(String::Strip($a), ';,', false, true)), 1);
  RETURN $replace2($replace(String::ToLower(String::JoinFromList($first_email, '')),'@yandex.ru'),'')
  ;
};
$PATH_ALL = $PATH_ROOT || "/union/all";
INSERT INTO $PATH_ALL WITH TRUNCATE
SELECT
  a.ClientID AS ClientID
, $normalize_ya_email(a.email) AS email
, a.dt AS dt
, a.Source AS Source
, s.dt_sent AS dt_sent
, s.campaign_id AS campaign_id
, IF(s.status_sent IS NULL, 0, 1) AS status_sent
, o.dt_open AS dt_open
, IF(o.status_open IS NULL, 0, 1) as status_open
, IF(u.status_unsub IS NULL, 0, 1) as status_unsub
FROM $PATH_ALL_EMAIL AS a
LEFT JOIN (
  SELECT
    recepient
  , iso_eventtime AS dt_sent
  , campaign_id,
  "1" AS status_sent
  FROM $PATH_SENT
  ) AS s
  ON a.email = s.recepient
LEFT JOIN (
  SELECT
    email
  , iso_eventtime AS dt_open
  , campaign_id
  , "1"  AS status_open
  FROM $PATH_OPENED
  WHERE event = 'opened'
  ) AS o
  ON a.email = o.email
 AND s.campaign_id = o.campaign_id
LEFT JOIN (
  SELECT
    email
  , iso_eventtime AS dt_unsub
  , campaign_id
  , "1"  AS status_unsub
  FROM $PATH_OPENED
  WHERE event = 'unsubscribed'
  ) AS u
  ON a.email = u.email
 AND s.campaign_id = u.campaign_id
ORDER BY
  ClientID
, status_sent
, status_open
, dt_sent
, Source
, status_unsub
;

/* 4. Из всех адресов далее выбирается самый актуальный адрес */
COMMIT;  -- (after all)

$addresses_unsubscribed = (
SELECT
  SUBSTRING(dt, 0, 10) AS dt
, email
, ClientID
, Source
FROM $PATH_ALL
WHERE status_unsub = 1 
  AND ClientID IS NOT NULL
);
$opened = (
SELECT
  ClientID
, max_by(SUBSTRING(dt, 0, 10), dt_open) AS dt
, max_by(email, dt_open) AS email
, max_by(Source, dt_open) AS source
FROM $PATH_ALL AS t1
LEFT ONLY JOIN $addresses_unsubscribed AS bl -- не берем адреса от которых отписались
  ON t1.ClientID = bl.ClientID
 AND t1.email = bl.email
WHERE status_open = 1
  AND ClientID IS NOT NULL
  AND status_unsub != 1
GROUP BY ClientID AS ClientID
);
$not_unsub = (
SELECT
  SUBSTRING(dt, 0, 10) AS dt
, email
, ClientID
, Source
FROM $PATH_ALL AS a
LEFT ONLY JOIN $opened AS bl
  ON a.ClientID = bl.ClientID
WHERE status_unsub != 1
  AND status_open != 1
  AND ClientID IS NOT NULL
);
$sent = (
SELECT DISTINCT email
FROM $PATH_ALL
WHERE status_sent = 1
 AND ClientID IS NOT NULL
);
-- не важно отписан или нет, логин остается адресом если других нет, даже если с него отписались
-- $login_addresses_unsubscribed = (
-- SELECT
--   SUBSTRING(dt, 0, 10) AS dt
-- , email
-- , ClientID
-- , Source
-- FROM $PATH_ALL
-- WHERE status_unsub = 1
--   AND Source = 'login'
--   AND ClientID IS NOT NULL
-- );
$login_default = (
SELECT DISTINCT email
FROM $PATH_ALL
WHERE Source = 'login' 
  AND ClientID IS NOT NULL
);
$space_upsale_recommendations = (
SELECT
  ClientID
, max_by(email, dt) AS email
, max(dt) AS dt
, max_by(Source, dt) AS source
FROM $not_unsub
WHERE Source IN ("space", "upsale", "recommendations")
GROUP BY ClientID
);
-- есть адрес для уведомлений
$notifications = (
SELECT
  ClientID
, max_by(a.email, a.dt) AS email
, max(a.dt) AS dt
, max_by(a.Source, a.dt) AS source
FROM $not_unsub AS a
LEFT ONLY JOIN $space_upsale_recommendations AS bl
  ON a.ClientID = bl.ClientID
WHERE Source IN ("notifications")
GROUP BY a.ClientID AS ClientID
);
$login = (
SELECT
  ClientID
, max_by(a.email, a.dt) AS email
, max(a.dt) AS dt
, max_by(a.Source, a.dt) AS source
FROM $not_unsub AS a
LEFT ONLY JOIN $space_upsale_recommendations AS sbl
  ON sbl.ClientID = a.ClientID
LEFT ONLY JOIN $notifications AS nbl
  ON nbl.ClientID = a.ClientID
WHERE Source NOT IN ("notifications","space", "upsale", "recommendations")
GROUP BY a.ClientID AS ClientID
);
-- если рассылки отправлялись и открывались, берем адрес из последнего 
-- открытия рассылки и максимально актуального источника
$PATH_ACTUAL_EMAILS = $PATH_ROOT || "/export/actual_emails";
INSERT INTO $PATH_ACTUAL_EMAILS WITH TRUNCATE
SELECT
  ClientID
, u.email AS email
, source
, dt
, is_confirmed
, IF(s.email IS NULL, 0, 1) AS status_sent
, IF(d.email IS NULL, 0, 1) AS is_login
FROM (
  SELECT ClientID, email, source, dt, 1 AS is_confirmed
  FROM $opened
    UNION ALL
  SELECT ClientID, email, source, dt, 0 AS is_confirmed,
  FROM $space_upsale_recommendations
    UNION ALL
  SELECT ClientID, email, source, dt, 0 AS is_confirmed,
  FROM $notifications
    UNION ALL
  SELECT ClientID, email, source, dt, 0 AS is_confirmed,
  FROM $login
) AS u
LEFT JOIN $sent AS s
  ON u.email = s.email
LEFT JOIN $login_default AS d
  ON u.email = d.email
ORDER BY ClientID
;

-- агентские представители
$agencies = (
SELECT ClientID
FROM `//home/direct/db/clients`
WHERE role = "agency"
);
$all_emails = (
SELECT ClientID, rep_type, email, source
FROM (
  SELECT ClientID, rep_type, email,  "notifications" AS source
  FROM `//home/direct/db/users`
  WHERE ClientID IN COMPACT $agencies
    AND email IS NOT NULL
    UNION ALL
  SELECT ClientID, rep_type, recommendations_email, "recommendations" AS source
  FROM `//home/direct/db/users`
  WHERE ClientID IN COMPACT $agencies
    AND email IS NOT NULL
    UNION ALL
  SELECT ClientID, email, SOURCE AS source
  FROM $PATH_LOGIN
  WHERE ClientID IN COMPACT $agencies
    AND email IS NOT NULL
  )
WHERE $normalize_email_safe(email) IS NOT NULL
GROUP BY ClientID, rep_type, $normalize_email_safe(email) AS email, source
);

$PATH_ALL_AGENCY_REP = $PATH_ROOT || "/union/all_agencies_representatives";
INSERT INTO $PATH_ALL_AGENCY_REP WITH TRUNCATE
SELECT
  a.ClientID AS ClientID
, nvl(a.rep_type, "other") AS rep_type
, a.email AS email
, a.source AS Source
, o.dt_open AS dt_open
, IF(o.status_open IS NULL, 0, 1) AS status_open
, IF(u.status_unsub IS NULL, 0, 1) AS status_unsub
, space.dt AS dt_space
, space.ClientID AS ClientID_space
, upsale.dt AS dt_upsale
, upsale.ClientID AS ClientID_upsale
FROM $all_emails AS a
LEFT JOIN (
  SELECT recepient, iso_eventtime AS dt_sent, campaign_id, "1" AS status_sent
  FROM $PATH_SENT
  ) AS s
  ON a.email = s.recepient
LEFT JOIN (
  SELECT email, iso_eventtime AS dt_open, campaign_id, "1"  AS status_open
  FROM $PATH_OPENED
  WHERE event = 'opened'
  ) AS o
  ON a.email = o.email
 AND s.campaign_id = o.campaign_id
LEFT JOIN (
  SELECT email, iso_eventtime AS dt_unsub, campaign_id, "1"  AS status_unsub
  FROM $PATH_OPENED
  WHERE event = 'unsubscribed'
  ) AS u
  ON a.email = u.email
 AND s.campaign_id = u.campaign_id
LEFT JOIN (
  SELECT max_by(ClientID, dt) AS ClientID, email, max(dt) AS dt
  FROM $PATH_SPACE
  GROUP BY email
  ) AS space
  ON a.email = space.email
LEFT JOIN (
  SELECT max_by(ClientID, dt) AS ClientID, email, max(dt) AS dt
  FROM $PATH_UPSALE
  GROUP BY email
  ) AS upsale
  ON a.email = upsale.email
WHERE NOT (
      Source = 'login'
  AND status_open = 0
  AND status_unsub = 0
  AND ClientID_space IS NULL
  AND ClientID_upsale IS NULL
)
GROUP BY
  a.ClientID AS ClientID
, a.rep_type AS rep_type
, a.email AS email
, a.source AS Source
, o.dt_open AS dt_open
, o.status_open
, u.status_unsub
, IF(o.status_open IS NULL, 0, 1) AS status_open
, IF(u.status_unsub IS NULL, 0, 1) AS status_unsub
, space.dt AS dt_space
, space.ClientID AS ClientID_space
, upsale.dt AS dt_upsale
, upsale.ClientID AS ClientID_upsale
ORDER BY
  ClientID
, status_open
, Source
, status_unsub
;

-- актуальные адреса для представителей РА ('chief', 'main', 'limited', 'other', '')
$PATH_ACTUAL_AGENCY_CHIEF = $PATH_ROOT || "/export/actual_emails_for_agencies_representatives_chief";
$PATH_ACTUAL_AGENCY_MAIN = $PATH_ROOT || "/export/actual_emails_for_agencies_representatives_main";
$PATH_ACTUAL_AGENCY_LIMITED = $PATH_ROOT || "/export/actual_emails_for_agencies_representatives_limited";
$PATH_ACTUAL_AGENCY_OTHER = $PATH_ROOT || "/export/actual_emails_for_agencies_representatives_other";
$PATH_ACTUAL_AGENCY_ANY = $PATH_ROOT || "/export/actual_emails_for_agencies_representatives";
$PATH_ACTUAL_AGENCY_ERROR = $PATH_ROOT || "/export/actual_emails_for_agencies_representatives_error";

DEFINE ACTION $calc_representatives($rep_type) AS
$DST_PATH =
  CASE $rep_type
    WHEN "chief" THEN $PATH_ACTUAL_AGENCY_CHIEF
    WHEN "main" THEN $PATH_ACTUAL_AGENCY_MAIN
    WHEN "limited" THEN $PATH_ACTUAL_AGENCY_LIMITED
    WHEN "other" THEN $PATH_ACTUAL_AGENCY_OTHER
    WHEN "" THEN $PATH_ACTUAL_AGENCY_ANY
    ELSE $PATH_ACTUAL_AGENCY_ERROR
  END;

$REP_TUP = IF($rep_type = "", AsList("chief", "main", "limited", "other"), AsList($rep_type,));


$addresses_unsubscribed = (
SELECT
  SUBSTRING(dt_open, 0, 10) as dt
, email
, ClientID
, Source
FROM $PATH_ALL_AGENCY_REP
WHERE status_unsub = 1
  AND ClientID IS NOT NULL
  AND rep_type IN $REP_TUP
);
$opened = (
SELECT
  ClientID
, max_by(SUBSTRING(dt_open, 0, 10), dt_open) AS dt
, max_by(email, dt_open) AS email
, max_by(Source, dt_open) AS source
FROM $PATH_ALL_AGENCY_REP AS t1
LEFT ONLY JOIN $addresses_unsubscribed AS bl
  ON t1.ClientID = bl.ClientID
 AND t1.email = bl.email
WHERE status_open = 1
  AND ClientID IS NOT NULL
  AND rep_type IN $REP_TUP
  AND status_unsub != 1
GROUP BY ClientID AS ClientID
);
$not_unsub = (
SELECT
  email
, ClientID
, rep_type
, Source
, dt_space
, dt_upsale
FROM $PATH_ALL_AGENCY_REP AS a
LEFT ONLY JOIN $opened AS bl
  ON a.ClientID = bl.ClientID
WHERE status_unsub != 1
  AND status_open != 1
  AND ClientID IS NOT NULL
  AND rep_type IN $REP_TUP
);
$space = (
SELECT
  ClientID
, max_by(email, dt_space) AS email
, max(dt_space) AS dt
, max_by(Source, dt_space) AS source
FROM $not_unsub
WHERE dt_space IS NOT NULL
GROUP BY ClientID
);
$upsale = (
SELECT
  ClientID
, max_by(email, dt_upsale) AS email
, max(dt_space) AS dt
, max_by(Source, dt_upsale) AS source
FROM $not_unsub
WHERE dt_upsale IS NOT NULL
  AND dt_space IS NULL
GROUP BY ClientID
);
$other = (
SELECT
  ClientID
, min_by(email, rep_type) AS email
, min_by(Source, rep_type) AS source
FROM $not_unsub AS nu
LEFT ONLY JOIN $space AS s
  ON nu.ClientID = s.ClientID
LEFT ONLY JOIN $upsale AS us
  ON nu.ClientID = us.ClientID
WHERE dt_upsale IS NULL
  AND dt_space IS NULL
GROUP by ClientID AS ClientID
);
$login_default = (
SELECT DISTINCT email
FROM $PATH_ALL_AGENCY_REP
WHERE Source = 'login'
  AND ClientID is not null
  AND rep_type IN $REP_TUP
);

INSERT INTO $DST_PATH WITH TRUNCATE
SELECT
  ClientID
, u.email AS email
, source
, dt
, is_confirmed
, $rep_type AS rep_type
, IF(d.email IS NULL, 0, 1) AS is_login
FROM (
  SELECT ClientID, email, source, dt, 1 AS is_confirmed
  FROM $opened
    UNION ALL
  SELECT ClientID, email, source, dt, 0 AS is_confirmed,
  FROM $space
    UNION ALL
  SELECT ClientID, email, source, dt, 0 AS is_confirmed,
  FROM $upsale
    UNION ALL
  SELECT ClientID, email, source, NULL AS dt, 0 AS is_confirmed,
  FROM $other
) AS u
LEFT JOIN $login_default AS d
  ON u.email = d.email
ORDER BY ClientID;
END DEFINE;
;

COMMIT;  -- after all_agencies_representatives
EVALUATE FOR $x IN ['chief', 'main', 'limited', 'other', '']
DO $calc_representatives($x);
; COMMIT;

-- актуальные адреса для представителей РА by rep_type
$PATH_ACTUAL_AGENCY_ROLES = $PATH_ROOT || "/export/actual_emails_for_agencies_representatives_with_roles";
INSERT INTO $PATH_ACTUAL_AGENCY_ROLES WITH TRUNCATE
SELECT
  ClientID
, email
, source
, dt
, is_confirmed
, rep_type
, is_login
FROM (
    SELECT ClientID, email, source, dt, rep_type, is_confirmed, is_login
    FROM $PATH_ACTUAL_AGENCY_CHIEF
      UNION ALL
    SELECT ClientID, email, source, dt,rep_type, is_confirmed, is_login
    FROM $PATH_ACTUAL_AGENCY_MAIN
      UNION ALL
    SELECT ClientID, email, source, dt, rep_type, is_confirmed, is_login
    FROM $PATH_ACTUAL_AGENCY_LIMITED
      UNION ALL
    SELECT ClientID, email, source, dt, rep_type, is_confirmed, is_login
    FROM $PATH_ACTUAL_AGENCY_OTHER
)
ORDER BY ClientID
;