#!/usr/bin/env python
# -*- coding: utf-8 -*-

import logging
import os

import requests
import json
import urllib.parse
import urllib3
import time
from datetime import timedelta, date, datetime
import dateutil.parser

from yt.wrapper.client import Yt
import yt

def login(session, login, password):
    session.post("https://passport.yandex-team.ru/auth", data={"login": login, "passwd": password})

def get_artmon_traffic2(session, period_start, period_end, group_context_type, group_sim_distance):
    URL = "https://artmon.bsadm.yandex-team.ru/cgi-bin/data.cgi?module=Traffic2"

    payload = {
        "period_start": period_start,
        "period_start_hour": "00",
        "period_end": period_end,
        "period_end_hour": "23",
        "compare_enabled": 0,
        "compare_start": period_start,
        "compare_start_hour": "00",
        "compare_end": period_end,
        "compare_end_hour": "23",
        "timegroup": "day",
        "periodicity": "undefined",
        "query": json.dumps({
            "filters": {
                "YandexFilter":"all",
                "OrderFilter":"all",
                "TypeFilter":"all",
                "ContextFilter":0,
                "group_context_type":[group_context_type],
                "UserFilter":"all",
                "detailed":0,
                "Slice":"preferreddeals",
                "relative":0,
                "dont_show_total":1,
                "device_type":"all",
                "group_device_type":None,
                "group_product_type":None,
                "product_type":"all",
                "distribution":1,
                "placeid_paid_filter":"542,1542",
                "placeid_filter":"542,1542",
                "pageid_filter":"",
                "impid_filter":"",
                "group_select_type":None,
                "group_sim_distance":[group_sim_distance],
                "group_turbo":None,
                "group_inapp":None,
                "group_ya_service":None,
                "group_ssp":None,
                "group_video":None,
                "group_video_kind":None,
                "group_video_format":None,
                "group_preferred_deals":None,
                "group_is_adfox":None,
                "exclude_pageid_filter":"",
                "cross_filter":"0",
                "undefined":"",
                "domain_ajax":"",
                "login_ajax":"",
                "failed_money":0,
                "show_visit_info":0,
                "show_ad_serp_hits":0,
                "not_only_good":0,
                "discount":0,
                "hide_ext":1,
                "logarithm_axis":0,
                "fraction_series":1,
                "fraction_series_shift":1,
                "ignore_fast":0,
                "fast_traffic":0,
                "clickhouse":0,
                "bsclickhouse":1,
                "fraud_series":0
            }
        }, separators=(',', ':')),
        #"filename_postfix": "_2020-05-01-00_2020-05-12-23",
        "output_format": "json",
        "dl": 1,
    }

    response = session.get(URL + "&" + urllib.parse.urlencode(payload), verify=False)
    return json.loads(response.content)

def main():
    urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
    logging.basicConfig(level=logging.DEBUG, format="[%(asctime)s] %(levelname)s %(thread)d %(message)s")

    GROUP_DYNAMIC               = "7"
    SD_DYNAMIC_QUERIES          = "700010"
    SD_DYNAMIC_LINKS_QUERIES    = "700050"
    SD_DYNAMIC_WM_CLUSTERS      = "700333"

    GROUP_SMART                 = "8"
    SD_SMART_LINKS_QUERIES      = "804050"
    SD_SMART_WM_CLUSTERS        = "800333"

    UNITS       = 30.0

    F_PERIOD    = "Period"
    F_D_700010  = "D_700010"    #dynamic queries
    F_D_700050  = "D_700050"    #dynamic links+queries
    F_D_700333  = "D_700333"    #dynamic wm_clusters
    F_S_804050  = "S_804050"    #smart links+queries
    F_S_800333  = "S_800333"    #smart wm_clusters
    F_TOTAL     = "Total"
    F_TOTAL_RUB = "TotalRUB"    #Total(y.e) * 30.0 (UNITS)

    DATE_WINDOW = 183
    DATE_DEFAULT = dateutil.parser.parse("2019-01-01").date()

    TABLE = "//home/webmaster/prod/analytics/catalogia/artmon"

    client = Yt(proxy="hahn.yt.yandex.net", config={"token_path": os.getenv("YT_TOKEN_PATH")})

    date_start = DATE_DEFAULT

    SOURCES = [
        (F_D_700010, GROUP_DYNAMIC, SD_DYNAMIC_QUERIES),
        (F_D_700050, GROUP_DYNAMIC, SD_DYNAMIC_LINKS_QUERIES),
        (F_D_700333, GROUP_DYNAMIC, SD_DYNAMIC_WM_CLUSTERS),
        (F_S_804050, GROUP_SMART,   SD_SMART_LINKS_QUERIES),
        (F_S_800333, GROUP_SMART,   SD_SMART_WM_CLUSTERS),
    ]

    rows = []
    if client.exists(TABLE):
        for line in client.read_table(TABLE, raw=False, format=yt.wrapper.JsonFormat()):
            date_start = max(date_start, dateutil.parser.parse(line[F_PERIOD]).date())
            rows.append(line)
    else:
        schema = [
            {"name": F_PERIOD,      "type": "string", "sort_order": "ascending"},
            {"name": F_D_700010,    "type": "double"},
            {"name": F_D_700050,    "type": "double"},
            {"name": F_D_700333,    "type": "double"},
            {"name": F_S_804050,    "type": "double"},
            {"name": F_S_800333,    "type": "double"},
            {"name": F_TOTAL,       "type": "double"},
            {"name": F_TOTAL_RUB,   "type": "double"},
        ]

        logging.info("catalogia, no statistics table, creating %s", TABLE)
        client.create("table", TABLE, attributes={"schema": schema})

    date_end = date_start + timedelta(DATE_WINDOW)
    date_end = min(date_end, date.today() - timedelta(1))

    if date_start >= date_end:
        return

    date_start  = str(date_start)
    date_end    = str(date_end)

    logging.info("updating, %s - %s", date_start, date_end)

    parsed = {}
    with client.Transaction():
        session = requests.Session()
        login(session, "robot-webmaster", "111")

        for field, group, sim_distance in SOURCES:
            logging.info("source, group=%s, sim_distance=%s", group, sim_distance)

            obj = get_artmon_traffic2(session, date_start, date_end, group, sim_distance)

            for row in obj["items"]["rows"]:
                if "cost" not in row or row["series_id"] != "bsclickhouse":
                    continue

                cost = float(row["cost"])
                period = row["utc"]
                period = datetime.fromtimestamp(period / 1000)
                period = str(period.date())
                #period = time.strftime("%Y-%m-%d", period)

                if period not in parsed:
                    parsed[period] = {
                        F_D_700010: 0.0,
                        F_D_700050: 0.0,
                        F_D_700333: 0.0,
                        F_S_804050: 0.0,
                        F_S_800333: 0.0,
                    }

                logging.info("source, %s %s %.3f", field, period, cost)
                parsed[period][field] = cost

        for period in sorted(parsed):
            pd = parsed[period]
            total = pd[F_D_700010] + pd[F_D_700050] + pd[F_D_700333] + pd[F_S_804050] + pd[F_S_800333]
            total_rub = total * UNITS
            rows.append({
                F_PERIOD:       period,
                F_D_700010:     pd[F_D_700010],
                F_D_700050:     pd[F_D_700050],
                F_D_700333:     pd[F_D_700333],
                F_S_804050:     pd[F_S_804050],
                F_S_800333:     pd[F_S_800333],
                F_TOTAL:        total,
                F_TOTAL_RUB:    total_rub
            })

        client.write_table(yt.wrapper.TablePath(TABLE, sorted_by=[F_PERIOD]), rows, raw=False)

    logging.info("done")


if __name__ == "__main__":
    main()
