-- OPERANALYTICS-1252
USE hahn;
PRAGMA AnsiInForEmptyOrNullableItemsCollections;

DECLARE $param_dict AS Dict<String, String>;

$base_dir = IF(
  $param_dict["branch"] = "prod",
  "//home/vipplanners/sow",
  "//home/vipplanners/sow_dev" || '/' || $param_dict["branch"]
);
-- input
$sprav = $base_dir || '/' || "dict/sources/sprav";
$sprav_banner_max_legal = $base_dir || '/' || "dict/sources/sprav_banner_max_legal";
$sprav_company_max_legal = $base_dir || '/' || "dict/sources/sprav_company_max_legal";
$client_main_domain = $base_dir || '/' || "dict/sources/client_main_domain";
-- output
$result = $base_dir || '/' || "dict/client_info.v4";

$client_wl = (
SELECT DISTINCT ClientID AS client_id
FROM `//home/direct/db/users`
WHERE statusBlocked == "No"
);

$client_account_crm = (
SELECT account_id, client_id
FROM `//home/crm/clients/account`
WHERE client_id IN COMPACT $client_wl
GROUP BY
  id AS account_id
, client_id
);

$NOW = CurrentUTCDatetime();

$first_email = ($emails_str) -> (String::SplitToList($emails_str, ";,", false, false)[0]);

$filter_value = ($xs) -> {
-- последние 10 уникальных критерией не старше двух лет
  $xs = ListSortDesc($xs, ($x) -> ($x.dt));
  RETURN ListTake(ListUniq(ListFlatMap(
    $xs,
    ($x) -> (IF($NOW - $x.dt < Interval("P730D"), $x.val))
  )), 10);
};

$client_contact_crm = (
SELECT
  client_id
, MAX_BY(a.phone_e164, AsTuple(a.ref_count, a.ref_last_dt, a.modified_on, a.id)) AS phone_last  -- ref_last_dt - Дата последней коммуникации с этим контактом
, MAX_BY($first_email(a.email), AsTuple(a.ref_count, a.ref_last_dt, a.modified_on, a.id)) AS email_last
, $filter_value(TOP_BY(<|val:a.phone_e164, dt:CAST(CAST(ref_last_dt AS String) AS Datetime)|>, a.ref_last_dt, 1000)) AS phone_top
, $filter_value(TOP_BY(<|val:$first_email(a.email), dt:CAST(ref_last_dt AS Datetime)|>, a.ref_last_dt, 1000)) AS email_top
FROM `//home/crm/contacts/account_kik` AS a
JOIN $client_account_crm AS c
  ON a.account_id == c.account_id
WHERE c.client_id > 0
GROUP BY c.client_id AS client_id
);

$client_contact_direct = (
  SELECT
    client_id
  , MAX_BY(phone, create_time) AS phone_last
  , MAX_BY($first_email(email), create_time) AS email_last
  FROM `//home/direct/db/users`
  WHERE ClientID > 0
    AND ClientID IN COMPACT $client_wl
  GROUP BY ClientID AS client_id
);

$client_contact_balance = (
SELECT
  client_id
, MAX_BY(phone, creation_dt) AS phone_last
, MAX_BY($first_email(email), creation_dt) AS email_last
FROM `//home/balance/prod/bo/t_client`
  WHERE id > 0
  AND id IN COMPACT $client_wl
GROUP BY id AS client_id
);

$client_contact_balance_person = (
SELECT
  client_id
, MAX_BY(phone, (dt, id)) AS phone_last
, MAX_BY($first_email(email), (dt, id)) AS email_last
, MAX_BY(inn, (dt, id)) AS inn_last
, $filter_value(TOP_BY(<|val:phone, dt:CAST(dt AS Datetime)|>, (dt, id), 1000)) AS phone_top
, $filter_value(TOP_BY(<|val:$first_email(email), dt:CAST(dt AS Datetime)|>, (dt, id), 1000)) AS email_top
, $filter_value(TOP_BY(<|val:inn, dt:CAST(dt AS Datetime)|>, (dt, id), 1000)) AS inn_top
FROM `//home/balance/prod/bo/t_person`
  WHERE id > 0
    AND hidden = 0
    AND client_id IN COMPACT $client_wl
GROUP BY client_id
);

$client_advisor = (
SELECT
  client_id
, curr_counterparty_geo.country_name AS curr_client_country
, curr_counterparty_reporting_tier.name AS curr_client_tier
, curr_counterparty_id
, curr_client_main_domain
, NVL(agency_id, 0) AS agency_id
FROM `//home/comdep-analytics/public/client_tiers/fact/latest_v2`
WHERE client_id IN COMPACT $client_wl
);

-- https://yql.yandex-team.ru/Operations/XyPT8Z3udmPuxCjuxQmMaP1Z4oZtlmGqZ4iqthxawxY=
$client_contact_sprav = (
SELECT
  client_id
,  String::SplitToList(list_phone_sprav, ',') AS phone_top
FROM $sprav
);

$datetime_parse = DateTime::Parse("%Y-%m-%d %H:%M:%S");

$client_contact_vcard = (
SELECT
  client_id
, MAX_BY(v.phone, (v.LastChange, v.vcard_id)) AS phone_last
, MAX_BY($first_email(v.contact_email), (v.LastChange, v.vcard_id)) AS email_last
, $filter_value(TOP_BY(<|val:String::RemoveAll(v.phone, '#'), dt:DateTime::MakeDatetime($datetime_parse(v.LastChange))|>, (v.LastChange, v.vcard_id), 1000)) AS phone_top
, $filter_value(TOP_BY(<|val:$first_email(v.contact_email), dt:DateTime::MakeDatetime($datetime_parse(v.LastChange))|>, (v.LastChange, v.vcard_id), 1000)) AS email_top
FROM `//home/direct/db/vcards` AS v
JOIN `//home/direct/db/campaigns` AS c
  ON v.cid = c.cid
WHERE c.ClientID IN COMPACT $client_wl
  AND c.ClientID > 0
GROUP BY c.ClientID AS client_id
);

PRAGMA File('libcrypta_identifier_udf.so', 'yt://hahn/home/crypta/public/udfs/libcrypta_identifier_udf.so');
PRAGMA Udf('libcrypta_identifier_udf.so');

$is_valid_inn = ($str_inn) -> {
  RETURN
    CASE
      WHEN $str_inn IS NULL THEN False
      WHEN $str_inn = '0000000000' THEN False
      WHEN length($str_inn) < 9 THEN False
    ELSE True END;
};

-- http://www.consultant.ru/cons/cgi/online.cgi?rnd=C4A70E496692A9AA3DD0F1281BD7118F&req=doc&base=LAW&n=311536&dst=100012&fld=134&stat=refcode%3D16876%3Bdstident%3D100012%3Bindex%3D0#25yk8t3tqrb
$is_valid_ogrn = ($str_ogrn) -> {
    RETURN
      CASE
        WHEN $str_ogrn IS NULL THEN False
        WHEN length($str_ogrn) < 13 THEN False
      ELSE True END;
};

$parse_inn = ($inn) -> {
    $str_inn = cast($inn as String);
    RETURN IF($is_valid_inn($str_inn), $str_inn);
};

$parse_ogrn = ($ogrn) -> {
    $str_ogrn = cast($ogrn as String);
    RETURN IF($is_valid_ogrn($str_ogrn), $str_ogrn);
};

INSERT INTO $result WITH TRUNCATE
SELECT
  client_id
, roll3y_client_main_domain
-- crm advisor
, curr_counterparty_id
, curr_client_main_domain
, agency_id
-- crm
, phone_last_crm.IsValid AS phone_last_crm_is_valid
, phone_last_crm.Normalize AS phone_last_crm
, email_last_crm.IsValid AS email_last_crm_is_valid
, email_last_crm.Normalize AS email_last_crm
, ListAny(ListMap(phone_top_crm, ($x)->($x.IsValid))) AS phone_top_crm_has_valid
, ListUniq(ListFlatMap(phone_top_crm, ($x)->($x.Normalize))) AS phone_top_crm
, ListAny(ListMap(email_top_crm, ($x)->($x.IsValid))) AS email_top_crm_has_valid
, ListUniq(ListFlatMap(email_top_crm, ($x)->($x.Normalize))) AS email_top_crm
-- direct
, phone_last_direct.IsValid AS phone_last_direct_is_valid
, phone_last_direct.Normalize AS phone_last_direct
, email_last_direct.IsValid AS email_last_direct_is_valid
, email_last_direct.Normalize AS email_last_direct
-- balance
, phone_last_balance.IsValid AS phone_last_balance_is_valid
, phone_last_balance.Normalize AS phone_last_balance
, email_last_balance.IsValid AS email_last_balance_is_valid
, email_last_balance.Normalize AS email_last_balance
-- balance_person
, phone_last_balance_person.IsValid AS phone_last_balance_person_is_valid
, phone_last_balance_person.Normalize AS phone_last_balance_person
, email_last_balance_person.IsValid AS email_last_balance_person_is_valid
, email_last_balance_person.Normalize AS email_last_balance_person
, $is_valid_inn(inn_last_balance_person) AS inn_last_balance_person_is_valid
, $parse_inn(inn_last_balance_person) AS inn_last_balance_person
, ListAny(ListMap(phone_top_balance_person, ($x)->($x.IsValid))) AS phone_top_balance_person_has_valid
, ListUniq(ListFlatMap(phone_top_balance_person, ($x)->($x.Normalize))) AS phone_top_balance_person
, ListAny(ListMap(email_top_balance_person, ($x)->($x.IsValid))) AS email_top_balance_person_has_valid
, ListUniq(ListFlatMap(email_top_balance_person, ($x)->($x.Normalize))) AS email_top_balance_person
, ListAny(ListMap(inn_top_balance_person, ($x)->($is_valid_inn($x)))) AS inn_top_balance_person_has_valid
, ListUniq(ListFlatMap(inn_top_balance_person, ($x)->($parse_inn($x)))) AS inn_top_balance_person
-- sprav
, ListAny(ListMap(phone_top_balance_person, ($x)->($x.IsValid))) AS phone_top_sprav_has_valid
, ListUniq(ListFlatMap(phone_top_balance_person, ($x)->($x.Normalize))) AS phone_top_sprav
, $is_valid_inn(inn_sprav_banner_max_cnt) AS inn_sprav_banner_max_cnt_is_valid
, $parse_inn(inn_sprav_banner_max_cnt) AS inn_sprav_banner_max_cnt
, $is_valid_ogrn(ogrn_sprav_banner_max_cnt) AS ogrn_sprav_banner_max_cnt_is_valid
, $parse_ogrn(ogrn_sprav_banner_max_cnt) AS ogrn_sprav_banner_max_cnt
, $is_valid_inn(inn_sprav_company_max_cnt) AS inn_sprav_company_max_cnt_is_valid
, $parse_inn(inn_sprav_company_max_cnt) AS inn_sprav_company_max_cnt
, $is_valid_ogrn(ogrn_sprav_company_max_cnt) AS ogrn_sprav_company_max_cnt_is_valid
, $parse_ogrn(ogrn_sprav_company_max_cnt) AS ogrn_sprav_company_max_cnt
-- vcard
, phone_last_vcard.IsValid AS phone_last_vcard_is_valid
, phone_last_vcard.Normalize AS phone_last_vcard
, email_last_vcard.IsValid AS email_last_vcard_is_valid
, email_last_vcard.Normalize AS email_last_vcard
, ListAny(ListMap(phone_top_vcard, ($x)->($x.IsValid))) AS phone_top_vcard_has_valid
, ListUniq(ListFlatMap(phone_top_vcard, ($x)->($x.Normalize))) AS phone_top_vcard
, ListAny(ListMap(email_top_vcard, ($x)->($x.IsValid))) AS email_top_vcard_has_valid
, ListUniq(ListFlatMap(email_top_vcard, ($x)->($x.Normalize))) AS email_top_vcard
FROM (
  SELECT
    wl.client_id AS client_id
  -- main domain
  , cmd.main_domain AS roll3y_client_main_domain
  -- crm advisor
  , adv.curr_counterparty_id AS curr_counterparty_id
  , adv.curr_client_main_domain AS curr_client_main_domain
  , nvl(adv.agency_id, 0) AS agency_id
  , nvl(adv.curr_client_country, "unknown") AS curr_client_country
  , nvl(adv.curr_client_tier, "unknown") AS curr_client_tier
  -- crm
  , Identifiers::Phone(crm.phone_last) AS phone_last_crm
  , Identifiers::Email(crm.email_last) AS email_last_crm
  , ListMap(crm.phone_top, Identifiers::Phone) AS phone_top_crm
  , ListMap(crm.email_top, Identifiers::Email) AS email_top_crm
  -- direct
  , Identifiers::Phone(dir.phone_last) AS phone_last_direct
  , Identifiers::Email(dir.email_last) AS email_last_direct
  -- balance
  , Identifiers::Phone(bal.phone_last) AS phone_last_balance
  , Identifiers::Email(bal.email_last) AS email_last_balance
  -- balance_person
  , Identifiers::Phone(blp.phone_last) AS phone_last_balance_person
  , Identifiers::Email(blp.email_last) AS email_last_balance_person
  , ListMap(blp.phone_top, Identifiers::Phone) AS phone_top_balance_person
  , ListMap(blp.email_top, Identifiers::Email) AS email_top_balance_person
  , blp.inn_last AS inn_last_balance_person
  , blp.inn_top AS inn_top_balance_person
  -- sprav
  , ListMap(sprav.phone_top, Identifiers::Phone) AS phone_top
  , sb.inn_max_cnt AS inn_sprav_banner_max_cnt
  , sb.ogrn_max_cnt AS ogrn_sprav_banner_max_cnt
  , sc.inn_max_cnt AS inn_sprav_company_max_cnt
  , sc.ogrn_max_cnt AS ogrn_sprav_company_max_cnt
  -- vcard
  , Identifiers::Phone(vcard.phone_last) AS phone_last_vcard
  , Identifiers::Email(vcard.email_last) AS email_last_vcard
  , ListMap(vcard.phone_top, Identifiers::Phone) AS phone_top_vcard
  , ListMap(vcard.email_top, Identifiers::Email) AS email_top_vcard
  FROM $client_wl AS wl
  LEFT JOIN $client_advisor AS adv
    ON wl.client_id = adv.client_id
  LEFT JOIN $client_contact_crm AS crm
    ON wl.client_id = crm.client_id
  LEFT JOIN $client_contact_direct AS dir
    ON wl.client_id = dir.client_id
  LEFT JOIN $client_contact_balance AS bal
    ON wl.client_id = bal.client_id
  LEFT JOIN $client_contact_balance_person AS blp
    ON wl.client_id = blp.client_id
  LEFT JOIN $client_contact_sprav AS sprav
    ON wl.client_id = sprav.client_id
  LEFT JOIN $sprav_banner_max_legal AS sb
    ON wl.client_id = sb.client_id
  LEFT JOIN $sprav_company_max_legal AS sc
    ON wl.client_id = sc.client_id
  LEFT JOIN $client_contact_vcard AS vcard
    ON wl.client_id = vcard.client_id
  LEFT JOIN $client_main_domain AS cmd
    ON wl.client_id = cmd.client_id
)
ORDER BY client_id
;
