import logging
import argparse
import datetime
import random
import time

from statface_client import StatfaceClient, STATFACE_PRODUCTION
from statface_client.report import StatfaceReportConfig
from startrek_client import Startrek
from yql.api.v1.client import YqlClient

from saas.tools.devops.lib23.service_token import ServiceToken
from saas.tools.devops.lib23.saas_entity import saas_service_iterator
from saas.tools.devops.lib23.saas_service import SaasService

from saas.tools.devops.lib23.deploy_manager_api import DeployManagerApiClient

import search.tools.devops.libs.utils as u

import yt.wrapper as yt

from infra.yasm.yasmapi import GolovanRequest

STARTREK_TOKEN = ServiceToken('startrek').get_token()
YQL_TOKEN = ServiceToken('yql').get_token()
STAT_TOKEN = ServiceToken('stat').get_token()
YT_TOKEN = ServiceToken('yt').get_token()
DEFAULT_SAMPLE_SIZE = 5

SAAS_DUTY_LOGINS = ['salmin', 'yrum', 'coffeeman', 'i024', 'anikella', 'saku', 'trofimenkov',
                    'derrior']

local_time = datetime.datetime.now()
one_day_before = local_time + datetime.timedelta(days=-1)
minus_one_week = local_time + datetime.timedelta(weeks=-1)

TEST_SUFFIX = ""  # "" or "-test"
# TEST_SUFFIX = "-test"
assert TEST_SUFFIX in ["", "-test"], "Either measurements or measurements-test allowed"
YT_MEASURMENTS_TABLE = '//home/saas/test-i024/measurements{}/duty-stat-{:4}-{:02}'.format(
    TEST_SUFFIX,
    local_time.year,
    local_time.month
)
YT_SLA_TABLE = '//home/saas/test-i024/measurements{}/sla-metrics-{:4}-{:02}'.format(
    TEST_SUFFIX,
    local_time.year,
    local_time.month
)
YT_CLUSTER = "arnold"

YT_SCHEMA = [
    {
        'name': "fielddate",
        'type': 'string'
    },
    {
        'name': "gencfg_allocated",
        'type': 'boolean'
    },
    {
        'name': "release_branch",
        'type': 'string'
    },
    {
        'name': "saas_ctype",
        'type': 'string'
    },
    {
        'name': "saas_service",
        'type': 'string'
    },
    {
        'name': "timestamp",
        'type': 'int64'
    },
    {
        'name': "yp_allocated",
        'type': 'boolean'
    }
]


def combine_chunks(path):

    yt_client = yt.YtClient(
        proxy=YT_CLUSTER,
        token=YT_TOKEN
    )
    # yt.spec_builders.MergeSpecBuilder.data_size_per_job = 50 * 1024 * 1024
    try:
        logging.info("Combining chunks in table {}".format(path))
        yt_client.run_merge(path, path, mode='auto', spec={'combine_chunks': True})
    except:
        logging.exception("Failed to combine chunks in table {}".format(path))


def branch_measure_name(branch_name):
    if branch_name is None:
        return 'none'
    return 'branch_{}'.format(branch_name).replace('.', '_')


def date_format(date):
    return '{}-{:02}-{:02}'.format(
        date.year,
        date.month,
        date.day
    )


def parse_cmd_args():
    description = 'Collect SaaS duty metrics and upload them to stat infrastructure'
    epilog = 'Usage : ./duty_stat --upload-to-stat --upload-to-yt'
    parser = argparse.ArgumentParser(description=description, epilog=epilog)

    parser.add_argument(
        '--upload-to-stat',
        default=False,
        action='store_true',
        help='upload collected metrics to statface'
    )

    parser.add_argument(
        '--upload-to-yt',
        default=False,
        action='store_true',
        help='upload per-service metrics to yt'
    )

    parser.add_argument(
        '--upload-sla-metrics',
        default=False,
        action='store_true',
        help='upload sla violation metrics to yt'
    )

    parser.add_argument(
        '--reverse-yt-ts',
        default=False,
        action='store_true',
        help="Don't use until you know what is it"
    )

    parser.add_argument(
        '--sample',
        default=False,
        const=DEFAULT_SAMPLE_SIZE,
        type=int,
        nargs='?',
        help='Analyze sample of N random services instead of all SaaS Cloud. Default N is {}'.format(
            DEFAULT_SAMPLE_SIZE)
    )

    parser.add_argument(
        '--saas-ctype',
        help='Analyze only one SaaS ctype'
    )

    parser.add_argument(
        '-d', '--debug',
        default=False,
        action='store_true'
    )

    return parser.parse_args()


def upload_metrics_to_stat(
        sms_qty,
        phone_qty,
        open_issues_qty,
        recently_closed_issues_qty,
        branch_statistics
):
    statface = StatfaceClient(host=STATFACE_PRODUCTION, oauth_token=STAT_TOKEN)
    report = statface.get_report('SAAS/devops')

    title = report.config.title
    dimensions = report.config.dimensions
    measures = report.config.measures
    for branch in branch_statistics:
        measures[branch_measure_name(branch)] = 'number'

    new_config = StatfaceReportConfig(title=title, measures=measures, dimensions=dimensions)
    new_config.check_valid()
    report.upload_config(new_config)

    report_data = {
        "fielddate": date_format(local_time),
        "saassup_tickets": open_issues_qty,
        'saassup_resolve_weekly': recently_closed_issues_qty,
    }
    for branch in branch_statistics:
        report_data[branch_measure_name(branch)] = branch_statistics[branch]

    logging.info("Report data : %s", report_data)
    report.upload_data(scale='daily', data=report_data)

    all_report_data = report.download_data(scale='daily')
    yesterday_str = date_format(one_day_before)
    yesterday_report_data = [data for data in all_report_data if data['fielddate'].startswith(yesterday_str)]
    if len(yesterday_report_data) == 1:
        report_data = yesterday_report_data[0]
        report_data.update({
            "fielddate": yesterday_str,
            'duty_sms': sms_qty,
            'duty_calls': phone_qty
        })

        logging.info("Report data : %s", report_data)
        report.upload_data(scale='daily', data=report_data)
    else:
        raise Exception("Can't obtain yesterday data from stat")


def obtain_duty_metrics():

    st = Startrek(useragent='saas/tools/devops/duty/duty_stat', token=STARTREK_TOKEN)

    open_issues = st.issues.find(filter={'queue': 'SAASSUP', 'resolution': 'empty()'})
    logging.info('Opened issues quantity: %s\nTotal: %s', open_issues, len(open_issues))

    recently_closed_issues = st.issues.find(filter={
        'queue': 'SAASSUP',
        'resolved': {
            'from': date_format(minus_one_week)
        }
    })
    logging.info('Recently closed issues quantity: %s\nTotal: %s', recently_closed_issues, len(recently_closed_issues))

    query_juggler_sms = """
PRAGMA yt.InferSchema;
SELECT
    `checks`, `message`, `method`, `timestamp`
FROM hahn.`statbox/juggler-banshee-log/{}`
WHERE `login` IN [{}] AND event_type = "message_processed" AND method = "sms"
LIMIT 100000;
    """.format(
        date_format(one_day_before),
        ", ".join(map(lambda x: '"{}"'.format(x), SAAS_DUTY_LOGINS))
    )

    query_juggler_phone = """
PRAGMA yt.InferSchema;
SELECT
    `checks`, `message`, `method`, `timestamp`
FROM hahn.`statbox/juggler-banshee-log/{}`
WHERE `login` IN [{}] AND event_type = "message_processed" AND method = "phone"
LIMIT 100000;
    """.format(
        date_format(one_day_before),
        ", ".join(map(lambda x: '"{}"'.format(x), SAAS_DUTY_LOGINS))
    )

    try:
        yql = YqlClient(token=YQL_TOKEN)

        get_juggler_sms = yql.query(query_juggler_sms, syntax_version=1)
        get_juggler_sms.run()
        juggler_sms = get_juggler_sms.get_results()
        sms_qty = len(juggler_sms.dataframe)

        get_juggler_phone = yql.query(query_juggler_phone, syntax_version=1)
        get_juggler_phone.run()
        juggler_phone = get_juggler_phone.get_results()
        phone_qty = len(juggler_phone.dataframe)
    except:
        sms_qty = 0
        phone_qty = 0
        logging.exception("Can't execute YQL")

    logging.info('Alerts quantity: %s sms, %s phone calls', sms_qty, phone_qty)

    return len(open_issues), len(recently_closed_issues), sms_qty, phone_qty


def main():
    args = parse_cmd_args()
    if args.debug:
        u.logger_setup(2)
        logging.debug(args)
    else:
        u.logger_setup(1)

    yt_client = yt.YtClient(proxy=YT_CLUSTER, token=YT_TOKEN)

    branch_statistics = {}
    branch_by_service = {}

    if args.saas_ctype:
        ctypes_to_analyze = [args.saas_ctype]
    else:
        dm = DeployManagerApiClient()
        ctypes_to_analyze = dm.stable_ctypes

    all_stable_services = [ss for ss in saas_service_iterator(saas_ctypes=ctypes_to_analyze)]
    random.shuffle(all_stable_services)

    if args.sample:
        saas_services_sample = all_stable_services[:args.sample]
        logging.warn("Using sample mode. Will process only %s", saas_services_sample)
    else:
        saas_services_sample = all_stable_services

    tsmap = {}
    if args.upload_sla_metrics:
        tsmap = ServiceWithSLA.generate_tsmap()

    for ss in saas_services_sample:
        if args.upload_to_yt or args.upload_to_stat:
            for ns in ss.nanny_services:

                has_groups_with_io_limits = ns.has_groups_with_io_limits()
                has_groups_without_io_limits = ns.has_groups_without_io_limits()

                yp_allocated = ns.is_yp_lite()
                gencfg_allocated = ns.is_gencfg_allocated()

            try:
                branch = ss.get_release_branch()
            except:
                branch = None

            try:
                has_backup = ss.has_backup()
            except:
                logging.exception("Can't determine if service %s/%s has backup", ss.ctype, ss.name)
                has_backup = False

            branch_by_service['{}/{}'.format(ss.ctype, ss.name)] = branch
            branch_statistics[branch] = branch_statistics.get(branch, 0) + 1

            collected_data = {
                "timestamp": int(local_time.timestamp()),
                "fielddate": date_format(local_time),
                "saas_ctype": ss.ctype,
                "saas_service": ss.name,
                "release_branch": branch,
                "yp_allocated": yp_allocated,
                "gencfg_allocated": gencfg_allocated,
                "has_backup": has_backup,
                "has_groups_with_io_limits": has_groups_with_io_limits,
                "has_groups_without_io_limits": has_groups_without_io_limits
            }
            logging.debug("Collected data for service %s/%s : %s", ss.ctype, ss.name, collected_data)

            if args.upload_to_yt:
                yt_client.write_table(
                    yt.TablePath(YT_MEASURMENTS_TABLE, append=True),
                    [collected_data],
                    raw=False
                )

        if args.upload_sla_metrics:
            ss = ServiceWithSLA(ss.ctype, ss.name)
            ts_begin, ts_end = ss._get_required_range(reversed=args.reverse_yt_ts, tsmap=tsmap)
            logging.info('Collecting SLA data on service %s/%s, %s-%s', ss.ctype, ss.name, ts_begin, ts_end)
            yt_data = ss.detect_sla_violations(ts_begin, ts_end, enforce_sla_unanswers=True)
            ss.upload_data_to_yt(yt_data)

    if args.upload_to_stat:
        logging.info('Collected branch statistics: %s', branch_statistics)
        open_issues_qty, recently_closed_issues_qty, sms_qty, phone_qty = obtain_duty_metrics()
        upload_metrics_to_stat(sms_qty, phone_qty, open_issues_qty, recently_closed_issues_qty, branch_statistics)

    combine_chunks(YT_MEASURMENTS_TABLE)
    combine_chunks(YT_SLA_TABLE)


class ServiceWithSLA(SaasService):
    sla_metrics = {
        'unanswers_5min_perc_crit': {
            "signal": "itype=searchproxy;ctype=prod:mul(100,div(saas_unistat-search-{ctype}-{service}-5xx_dmmm, sum(saas_unistat-search-{ctype}-{service}-2xx_dmmm, const(1))))",
            "severity": 'crit'
        },
        'unanswers_5min_perc_warn': {
            "signal": "itype=searchproxy;ctype=prod:mul(100,div(saas_unistat-search-{ctype}-{service}-5xx_dmmm, sum(saas_unistat-search-{ctype}-{service}-2xx_dmmm, const(1))))",
            "severity": 'warn'
        },
        "search_q_999_ms": {
            "signal": "itype=searchproxy;ctype=prod:quant(saas_unistat-times-{ctype}-{service}_dhhh,999)",
            "severity": "warn"
        },
        'search_q_99_ms': {
            "signal": "itype=searchproxy;ctype=prod:quant(saas_unistat-times-{ctype}-{service}_dhhh,99)",
            "severity": "warn"
        },
        # "unanswers": {
        #         "signal": "itype=searchproxy;ctype=prod:perc(saas_unistat-search-{ctype}-5xx_dmmm, saas_unistat-search-{ctype}-2xx_dmmm)"
        # }

        'search_rps': {
            'signal': 'itype=searchproxy;ctype=prod:div(hcount(saas_unistat-times-{ctype}-{service}_dhhh, 0, inf), normal())',
            'severity': 'user'
        },
        # 'total_index_size_bytes': {'severity': 'user'},
        # 'maxdocs': {'severity': 'user'}
    }

    additional_signals = {
        'total_rps': 'itype=searchproxy;ctype=prod:div(saas_unistat-search-{ctype}-{service}-2xx_dmmm, normal())'
    }

    MAX_SIGNALS_COUNT = 100
    DATA_LIFETIME_IN_SEC = 30 * 24 * 60 * 60

    @classmethod
    def sla_metrics_yt_table_schema(cls):
        schema = [
            {'name': 'service_id', 'type': 'string', 'required': True},
            {'name': 'service_weight', 'type': 'uint8', 'required': True},
            {'name': 'timestamp', 'type': 'datetime', 'required': True}
        ]
        severities = []
        for metric in cls.sla_metrics:
            schema.extend([
                {'name': '{}_violation'.format(metric), 'type': 'boolean', 'required': False},
            ])
            if cls.sla_metrics[metric]['severity'] not in severities:
                severities.append(cls.sla_metrics[metric]['severity'])
        for severity in severities:
            schema.extend([
                {'name': '{}_problems'.format(severity), 'type': 'uint8', 'required': False},
            ])
        for signal in cls.additional_signals:
            schema.extend(
                [{'name': signal, 'type': 'double', 'required': False}]
            )
        return schema

    @classmethod
    def generate_tsmap(cls):
        tsmap = {}
        yql = YqlClient(token=YQL_TOKEN)
        yql_query = """
USE arnold;
PRAGMA yt.InferSchema = '1';
SELECT
    `service_id`,
    MAX(`timestamp`) AS max_ts,
    MIN(`timestamp`) AS min_ts,
FROM LIKE(`//home/saas/test-i024/measurements`, "sla-metrics-%")
GROUP BY `service_id`
        """
        range_query = yql.query(yql_query, syntax_version=1)
        range_query.run()
        df = range_query.get_results().full_dataframe
        for _, row in df.iterrows():
            tsmap[row['service_id']] = (row['min_ts'].timestamp(), row['max_ts'].timestamp())

        return tsmap

    def __init__(self, ctype, name):
        super(ServiceWithSLA, self).__init__(ctype, name)

        self.signals = []
        for pattern in self.sla_metrics:
            threshold = self.sla_info.get(pattern, None)
            if threshold:
                signal = self.sla_metrics[pattern]['signal'].format(ctype=self.ctype, service=self.name)
                self.signals.append({
                    'signal': signal,
                    'threshold': threshold,
                    'label': pattern,
                    'severity': self.sla_metrics[pattern]['severity']
                })

    def __str__(self):
        return '/'.join([self.ctype, self.name])

    @property
    def id(self):
        return str(self)

    @property
    def weight(self):
        return self.sla_info.get('service_weight', 1)

    def fetch_golovan_data(self, ts_begin, ts_end, period=300, attempts=5):
        signals = [s['signal'] for s in self.signals]
        for signal in self.additional_signals.values():
            signals.append(signal.format(ctype=self.ctype, service=self.name))

        logging.debug(signals)

        for attempt in range(attempts):
            result = {}
            try:
                for ts, values in GolovanRequest('ASEARCH', period, ts_begin, ts_end - 1, signals, explicit_fail=True, max_retry=10, retry_delay=0.5, read_from_stockpile=True):
                    result[ts] = values
                    logging.debug(values)
                return result
            except Exception as e:
                logging.exception('golovan exception: {}'.format(e))
        raise Exception('Cannot get data form golovan')

    def detect_sla_violations(self, ts_begin=None, ts_end=None, enforce_sla_unanswers=False):

        if ts_begin is None and ts_end is None:
            ts_begin, ts_end = self._get_required_range(reversed=False)

        sla_table = {'unanswers_5min_perc_crit': 1, 'unanswers_5min_perc_warn': 0.1}
        sla_table.update(self.sla_info)

        yt_data = []
        golovan_data = self.fetch_golovan_data(ts_begin, ts_end)
        for ts, data in golovan_data.items():
            # data: {signal : value}
            yt_row = {
                'service_id': self.id,
                'service_weight': self.weight,
                'timestamp': ts
            }
            severity_problems = {'crit_problems': 0, 'warn_problems': 0, 'user_problems': 0}

            for s in self.signals:
                value = data[s['signal']]
                label = s['label']
                severity = s['severity']

                if sla_table.get(label, None):

                    is_violated = value is not None and value > self.sla_info[label]
                    yt_row['{}_violation'.format(label)] = is_violated

                    severity_key = severity + '_problems'
                    if severity_key not in severity_problems:
                        severity_problems[severity_key] = 0
                    if is_violated:
                        severity_problems[severity_key] += 1

            for s in self.additional_signals:
                yt_row.update({s: data[self.additional_signals[s].format(ctype=self.ctype, service=self.name)]})

            yt_row.update(severity_problems)
            logging.debug(yt_row)
            yt_data.append(yt_row)

        return yt_data

    def upload_data_to_yt(self, yt_data):

        yt_client = yt.YtClient(proxy=YT_CLUSTER, token=YT_TOKEN)
        if not yt_client.exists(YT_SLA_TABLE):
            yt_client.create(
                'table',
                path=YT_SLA_TABLE,
                recursive=True,
                ignore_existing=False,
                attributes={'schema': self.sla_metrics_yt_table_schema()}
            )
        yt_client.write_table(
            yt.TablePath(
                YT_SLA_TABLE,
                append=True,
            ),
            yt_data,
            raw=False
        )
        # yt_client.alter_table(YT_SLA_TABLE, schema=self.sla_metrics_yt_table_schema())

    def _get_required_range(self, reversed=False, tsmap={}):
        if self.id not in tsmap:
            yql = YqlClient(token=YQL_TOKEN)
            yql_query = """
PRAGMA yt.InferSchema = '1';
SELECT
    MAX(`timestamp`) AS max_ts,
    MIN(`timestamp`) AS min_ts,
FROM arnold.`{}`
WHERE `service_id` = "{}"
            """.format(YT_SLA_TABLE, self.id)

            range_query = yql.query(yql_query, syntax_version=1)
            range_query.run()
            try:
                results = range_query.get_results()
                max_ts, min_ts = tuple(results.full_dataframe.values[0])
                if min_ts is not None and max_ts is not None:
                    max_ts = max_ts.timestamp()
                    min_ts = min_ts.timestamp()
            except AttributeError as e:
                logging.exception("Probably bad or empty table. Error: {}".format(e))
                max_ts, min_ts = None, None
        else:
            min_ts, max_ts = tsmap[self.id]

        period = 300

        if max_ts is None and min_ts is None:
            ts_end = time.time() - 3 * period
            ts_begin = ts_end - 24 * 3600

        elif not reversed:
            ts_begin = max_ts
            ts_end = min(max_ts + 30 * 3600, time.time() - 3 * period)

        else:
            ts_end = min_ts - period
            ts_begin = ts_end - 24 * 3600

        return ts_begin, ts_end


if __name__ == '__main__':
    main()
