# -*- coding: utf-8 -*-
"""
Created on Nov 8, 2013

@author: noob
"""
import logging
from collections import OrderedDict
from http.client import responses

from os import strerror

from common.models import Server
from common.util.meta import escape_string
from common.util.aggregators import MonitoringAggregator, RTDetailsAggregator, ProtoCodesAggregator, \
    NetCodesAggregator, RTHistogramsAggregator
from common.util.decorators import memoized_property
from monitoring.models import Metric
from .plots_base import Table


class CasesDistTable(Table):
    title = 'Распределение ответов по тэгам'

    def get_columns(self):
        """
        returns list of http codes and it's count
        """
        data = OrderedDict()
        try:
            sql = '''
                select round(toFloat32(sum(resps)), 3), 
                round(toFloat64(sumArray([connect_time_sum, send_time_sum, latency_sum, receive_time_sum]))/1000/toFloat32(sum(resps)), 3),
                round(toFloat64(sum(connect_time_sum))/1000/toFloat32(sum(resps)), 3), 
                round(toFloat64(sum(send_time_sum))/1000/toFloat32(sum(resps)), 3), 
                round(toFloat64(sum(latency_sum))/1000/toFloat32(sum(resps)), 3), 
                round(toFloat64(sum(receive_time_sum))/1000/toFloat32(sum(resps)), 3)
                from loaddb.rt_microsecond_details_buffer
                where job_id={job} 
                and job_date = toDate({job_date})
                group by tag 
                order by tag
            '''
            avg_times_data = self.ch_client.select(sql, query_params=self.job_obj.basic_query_params)
            cases = [''] + self.job_obj.cases
            for i in range(len(cases)):
                case = cases[i]
                data[case] = {}
                try:
                    data[case]['count'] = avg_times_data[i][0]
                except IndexError:
                    data[case]['count'] = 0.0
                try:
                    data[case]['expect'] = avg_times_data[i][1]
                except IndexError:
                    data[case]['expect'] = 0.0
                try:
                    data[case]['connect_time'] = avg_times_data[i][2]
                except IndexError:
                    data[case]['connect_time'] = 0.0
                try:
                    data[case]['send_time'] = avg_times_data[i][3]
                except IndexError:
                    data[case]['send_time'] = 0.0
                try:
                    data[case]['latency'] = avg_times_data[i][4]
                except IndexError:
                    data[case]['latency'] = 0.0
                try:
                    data[case]['receive_time'] = avg_times_data[i][5]
                except IndexError:
                    data[case]['receive_time'] = 0.0

            # form data
            columns_data = [
                {'weight': 'bold', 'align': 'left', 'title': ' ',
                 'values': ['Overall'] + [key for key in list(data.keys()) if key]},
                {'align': 'right', 'title': 'Count', 'values': [data[key]['count'] for key in list(data.keys())]},
                {'align': 'right', 'title': 'Percent',
                 'values': ['100.000%'] + self.count_percentage([data[key]['count'] for key in list(data.keys()) if key])},
                {'align': 'right', 'title': 'Overall Time',
                 'values': ['{} ms'.format(data[key]['expect']) for key in list(data.keys())]},
                {'align': 'right', 'title': 'Connect Time',
                 'values': ['{} ms'.format(data[key]['connect_time']) for key in list(data.keys())]},
                {'align': 'right', 'title': 'Send Time',
                 'values': ['{} ms'.format(data[key]['send_time']) for key in list(data.keys())]},
                {'align': 'right', 'title': 'Latency',
                 'values': ['{} ms'.format(data[key]['latency']) for key in list(data.keys())]},
                {'align': 'right', 'title': 'Receive Time',
                 'values': ['{} ms'.format(data[key]['receive_time']) for key in list(data.keys())]},
            ]
            return columns_data
        except:
            logging.exception('Could not get cases resps data for job {} due to:'.format(self.job))
            return None


class CasesNetCodesTable(Table):
    title = 'Сетевые коды по тэгам'

    def get_columns(self):
        """
        returns list of http codes and it's count
        """
        columns_data = [{'weight': 'bold', 'align': 'left', 'title': ' ',
                         'values': ['Overall'] + self.job_obj.cases}]
        sql = '''select code, tag, 
            toFloat32(sum(cnt))
            from loaddb.net_codes_buffer
            where job_id={job}
            and job_date = toDate({job_date})
            group by code, tag
            order by code, tag
            '''
        data = self.ch_client.select(sql, query_params=self.job_obj.basic_query_params)
        net_codes = sorted(set(c[0] for c in data))
        i = 0
        for code in net_codes:
            column = {'align': 'right', 'title': str(code), 'values': []}
            code_values = {}
            while i < len(data) and data[i][0] == code:
                code_values[data[i][1]] = data[i][2]
                i += 1
            for tag in [''] + self.job_obj.cases:
                column['values'].append(code_values.get(tag, 0))
            columns_data.append(column)
        return columns_data


class CasesHttpCodesTable(Table):
    title = 'HTTP коды по тэгам'

    def get_columns(self):
        """
        returns list of http codes and it's count
        """
        columns_data = [{'weight': 'bold', 'align': 'left', 'title': ' ',
                         'values': ['Overall'] + self.job_obj.cases}]
        sql = '''select code, tag, 
            toFloat32(sum(cnt))
            from loaddb.proto_codes_buffer
            where job_id={job}
            and job_date = toDate({job_date})
            group by code, tag
            order by code, tag
            '''
        data = self.ch_client.select(sql, query_params=self.job_obj.basic_query_params)
        http_codes = sorted(set(c[0] for c in data))
        i = 0
        for code in http_codes:
            column = {'align': 'right', 'title': str(code), 'values': []}
            code_values = {}
            while i < len(data) and data[i][0] == code:
                code_values[data[i][1]] = data[i][2]
                i += 1
            for tag in [''] + self.job_obj.cases:
                column['values'].append(code_values.get(tag, 0))
            columns_data.append(column)
        return columns_data


class CasesCumulativeQuantilesTable(Table):
    title = 'Кумулятивные квантили по тэгам'
    quantiles = '99%', '98%', '95%', '90%', '85%', '80%', '75%', '50%'

    def get_columns(self):
        """
        returns list of http codes and it's count
        """
        columns_data = [
            {'weight': 'bold', 'align': 'left', 'title': ' ',
             'values': ['Overall'] + self.job_obj.cases},
        ]
        sql = '''select 
            quantilesExactWeighted(0.5, 0.75, 0.8, 0.85, 0.9, 0.95, 0.98, 0.99)(bin, cnt)
            from loaddb.rt_microsecond_histograms_buffer
            where job_id={job}
            and job_date = toDate({job_date})
            group by tag
            order by tag
            '''
        tags_data = self.ch_client.select(sql, query_params=self.job_obj.basic_query_params)

        for i in range(len(self.quantiles)):
            columns_data.append({'align': 'right',
                                 'title': self.quantiles[i],
                                 'values': [str(float(value[0][::-1][i])/1000) + ' ms' for value in tags_data]})
        return columns_data


class HttpDistTable(Table):
    title = 'Распределение HTTP кодов'

    def get_columns(self):
        """
        returns list of dicts with column settings and data
        """
        aggregator = ProtoCodesAggregator(self.job_obj, self.tag)
        data = aggregator.get_raw_data()

        count_data = [value[1] for value in data]
        http_codes_explained = ['{} - {}'.format(value[0], responses.get(int(value[0]), '???')) for value in data]
        columns = [
            {'weight': 'bold', 'align': 'left', 'title': ' ', 'values': http_codes_explained},
            {'align': 'right', 'title': 'Count', 'values': count_data},
            {'align': 'right', 'title': 'Percent', 'values': aggregator.count_percentage(count_data)}
        ]
        return columns


class NetDistTable(Table):
    title = 'Распределение сетевых кодов'

    def get_columns(self):
        """
        returns list of dicts with column settings and data
        """
        aggregator = NetCodesAggregator(self.job_obj, self.tag)
        data = aggregator.get_raw_data()

        count_data = [value[1] for value in data]
        net_codes_explained = ['{} - {}'.format(value[0], strerror(int(value[0]))) for value in data]
        columns = [
            {'weight': 'bold', 'align': 'left', 'title': ' ', 'values': net_codes_explained},
            {'align': 'right', 'title': 'Count', 'values': count_data},
            {'align': 'right', 'title': 'Percent', 'values': aggregator.count_percentage(count_data)}
        ]
        return columns


class TimesDistTable(Table):
    title = 'Распределение времен ответа'

    def get_columns(self):
        """
        returns list of dicts with column settings and data
        """
        aggregator = RTHistogramsAggregator(self.job_obj, self.tag)
        data = aggregator.get_raw_data()

        time_to_column = {'weight': 'bold', 'align': 'right', 'title': ' ',
                          'values': [value[0] for value in reversed(data)]}
        time_count_column = {'align': 'right', 'title': 'Count', 'values': [value[1] for value in reversed(data)]}
        time_percentage_column = {'align': 'right', 'title': 'Percent',
                                  'values': aggregator.count_percentage([value[1] for value in reversed(data)])}
        time_quantiles_column = {'align': 'right', 'title': 'Quantile',
                                 'values': aggregator.count_quantiles([value[1] for value in reversed(data)])}

        return [time_to_column, time_count_column, time_percentage_column, time_quantiles_column]


class QuantilesCumulativeTable(Table):
    title = 'Кумулятивные квантили'
    rows = '99%', '98%', '95%', '90%', '85%', '80%', '75%', '50%'

    def get_columns(self):
        """
        returns list of dicts with column settings and data
        """
        sql = '''
            select 
            quantilesExactWeighted(0.5, 0.75, 0.8, 0.85, 0.9, 0.95, 0.98, 0.99)(bin, cnt)
            from loaddb.rt_microsecond_histograms_buffer
            where job_id={job} 
            and job_date = toDate({job_date}) 
        '''
        if self.job_obj.multitag and self.tag:
            sql += 'and tag in ({cases_with_tag}) '
        else:
            sql += '''and tag='{tag}'  '''
        query_params = self.job_obj.basic_query_params.copy()
        query_params.update({
            'cases_with_tag': ','.join(
                ["'{}'".format(escape_string(case)) for case in self.job_obj.cases if self.tag in case.split('|')]),
            'tag': self.tag,
        })
        raw_data = self.ch_client.select(sql, query_params=query_params)
        quantiles = [float(q)/1000 for q in reversed(raw_data[0][0])] \
            if raw_data and raw_data[0] else [0] * len(self.rows)

        data = [{'weight': 'bold', 'align': 'right', 'title': ' ', 'values': self.rows},
                {'align': 'right', 'title': 'Quantile', 'values': quantiles}]
        return data


class AggregatesTable(Table):
    title = 'Агрегаты'

    def get_columns(self):
        """
        returns list of dicts with column settings and data
        """
        avg_values = []
        stddev_values = []
        min_values = []
        max_values = []

        aggregator = RTDetailsAggregator(self.job_obj, self.tag)

        for param in self.params:
            values = aggregator.get_param_data(param)
            avg_values.append(values[0] or 0)
            stddev_values.append(values[1] or 0)
            min_values.append(values[2] or 0)
            max_values.append(values[3] or 0)
        data = [
            {
                'weight': 'bold', 'align': 'left', 'title': ' ',
                'values': [self.params_dict[param] for param in self.params],
            },
            {'align': 'right', 'title': 'Minimum', 'values': min_values},
            {'align': 'right', 'title': 'Average', 'values': avg_values},
            {'align': 'right', 'title': 'Maximim', 'values': max_values},
            {'align': 'right', 'title': 'StdDev', 'values': stddev_values},
        ]
        return data

    @property
    def params(self):
        return list(self.params_dict.keys())

    @property
    def params_dict(self):
        return OrderedDict((
            ('resps', 'Resp per sec'),
            ('expect', 'Overall Time'),
            ('connect_time', 'Connect Time'),
            ('send_time', 'Send Time'),
            ('latency', 'Latency'),
            ('receive_time', 'Receive Time'),
            ('threads', 'Threads'),
        ))


class MonitoringAggregatesTable(Table):
    title = 'Агрегаты мониторинга'

    @property
    def target_n(self):
        return int(self.raw_case)
    
    @memoized_property
    def target_obj(self):
        return Server.objects.get(n=self.target_n)

    def get_columns(self):
        """
        returns list of dicts with column settings and data
        """
        aggregator = MonitoringAggregator(self.job_obj)
        data = aggregator.aggregate()

        target_hosts = [self.target_obj.dsc or self.target_obj.host] * len(self.sorted_target_metrics)
        metric_groups = self.metric_groups
        metric_names = self.metric_names
        values = data[self.target_obj.host]
        avg_values = [values[metric.code]['average'] for metric in self.sorted_target_metrics]
        stddev_values = [values[metric.code]['stddev'] for metric in self.sorted_target_metrics]
        min_values = [values[metric.code]['minimum'] for metric in self.sorted_target_metrics]
        max_values = [values[metric.code]['maximum'] for metric in self.sorted_target_metrics]
        median_values = [values[metric.code]['median'] for metric in self.sorted_target_metrics]
        data = [
            {'align': 'left', 'title': 'Target', 'values': target_hosts},
            {'align': 'left', 'title': 'Metrics group', 'values': metric_groups},
            {'align': 'left', 'title': 'Metric', 'values': metric_names},
            {'align': 'right', 'title': 'Minimum', 'values': min_values},
            {'align': 'right', 'title': 'Average', 'values': avg_values},
            {'align': 'right', 'title': 'Median', 'values': median_values},
            {'align': 'right', 'title': 'Maximum', 'values': max_values},
            {'align': 'right', 'title': 'StdDev', 'values': stddev_values},
        ]
        return data

    @memoized_property
    def sorted_target_metrics(self):
        try:
            sql = '''
                select metric_name
                from loaddb.monitoring_verbose_data_buffer
                where job_id={job}
                and job_date=toDate({job_date})
                and target_host='{target}'
                group by metric_name
            '''
            query_params = self.job_obj.basic_query_params.copy()
            query_params['target'] = self.target_obj.host
            fetched_data = self.ch_client.select(sql, query_params=query_params)
            metric_ids = [metric_id[0] for metric_id in fetched_data]
            sorted_target_metrics = sorted(
                (Metric.objects.get_or_create(code=m)[0] for m in metric_ids),
                key=lambda m: m.code, reverse=False
            )
            return sorted_target_metrics
        except:
            logging.exception('Could not get sorted_target_metrics for job {} due to:'.format(self.job_obj.n))
            return []

    @memoized_property
    def metric_groups(self):
        """
        returns list of values
        """
        try:
            metric_groups = []
            for metric_obj in self.sorted_target_metrics:
                metric_groups.append(metric_obj.code.split(':')[0].split('_')[0])  # ':' for custom
            return metric_groups
        except:
            logging.exception('Could not get metric groups for monitoring aggregates for job {}'.format(self.job_obj.n))
            return []

    @memoized_property
    def metric_names(self):
        """
        returns list of values
        """
        try:
            metric_names = []
            for metric_obj in self.sorted_target_metrics:
                if ':' not in metric_obj.code:
                    metric_names.append(metric_obj.code[metric_obj.code.find('_') + 1:])
                else:
                    metric_names.append(metric_obj.code[metric_obj.code.find(':') + 1:])
            return metric_names
        except:
            logging.exception('Could not get metric names for monitoring aggregates for job {}'.format(self.job_obj.n))
            return []


class MultitagTable(Table):
    delimiter = '|'

    @memoized_property
    def tags(self):
        """
        multitags are got from cases where there is a delimiter;
        some responses can have more then 1 tag;
        i.e. case == 'tag1<delimiter>tag2'
        """
        tags = []
        for case in self.job_obj.cases:
            tags.extend(case.split(self.delimiter))
        return sorted(set(tags))


class MultitagDistTable(MultitagTable):
    title = 'Распределение ответов по тэгам'

    def get_tag_avg_times(self, tag):
        """
        weighs times on the quantity of responses
        """
        try:
            sql = '''
                select toFloat32(sum(resps)), 
                toFloat64(sumArray([connect_time_sum, send_time_sum, latency_sum, receive_time_sum]))/1000/toFloat32(sum(resps)),
                toFloat64(sum(connect_time_sum))/1000/toFloat32(sum(resps)),
                toFloat64(sum(send_time_sum))/1000/toFloat32(sum(resps)),
                toFloat64(sum(latency_sum))/1000/toFloat32(sum(resps)),
                toFloat64(sum(receive_time_sum))/1000/toFloat32(sum(resps))
                from loaddb.rt_microsecond_details_buffer
                where job_id={job} 
                and job_date = toDate({job_date})
                and tag in ({cases_with_tag})
            '''
            query_params = self.job_obj.basic_query_params.copy()
            query_params.update({
                'cases_with_tag': ','.join("'{}'".format(escape_string(case)) for case in self.job_obj.cases if
                                           tag in case.split(self.delimiter)),
            })
            fetched_data = self.ch_client.select(sql, query_params=query_params)
            resps_count = sum(value[0] for value in fetched_data)
            data = [
                resps_count,
                round(sum(value[1] * value[0] for value in fetched_data) / resps_count, 3),
                round(sum(value[2] * value[0] for value in fetched_data) / resps_count, 3),
                round(sum(value[3] * value[0] for value in fetched_data) / resps_count, 3),
                round(sum(value[4] * value[0] for value in fetched_data) / resps_count, 3),
                round(sum(value[5] * value[0] for value in fetched_data) / resps_count, 3),
            ]
            return data
        except:
            logging.exception("Could not get avg times for {} for job {} for tag {}, due to:"
                              .format(self.__class__.__name__, self.job_obj.n, tag))
            return []

    def get_columns(self):
        """
        returns list of http codes and it's count
        """
        data = OrderedDict()
        for tag in self.tags:
            data[tag] = {}
            avg_times_data = self.get_tag_avg_times(tag)
            try:
                data[tag]['count'] = avg_times_data[0]
            except:
                data[tag]['count'] = 0.0
            try:
                data[tag]['expect'] = avg_times_data[1]
            except:
                data[tag]['expect'] = 0.0
            try:
                data[tag]['connect_time'] = avg_times_data[2]
            except:
                data[tag]['connect_time'] = 0.0
            try:
                data[tag]['send_time'] = avg_times_data[3]
            except:
                data[tag]['send_time'] = 0.0
            try:
                data[tag]['latency'] = avg_times_data[4]
            except:
                data[tag]['latency'] = 0.0
            try:
                data[tag]['receive_time'] = avg_times_data[5]
            except:
                data[tag]['receive_time'] = 0.0

        # form data
        columns_data = [
            {'weight': 'bold', 'align': 'left', 'title': ' ',
             'values': [key for key in list(data.keys()) if key]},
            {'align': 'right', 'title': 'Count', 'values': [data[key]['count'] for key in list(data.keys())]},
            {'align': 'right', 'title': 'Overall Time',
             'values': ['{} ms'.format(data[key]['expect']) for key in list(data.keys())]},
            {'align': 'right', 'title': 'Connect Time',
             'values': ['{} ms'.format(data[key]['connect_time']) for key in list(data.keys())]},
            {'align': 'right', 'title': 'Send Time',
             'values': ['{} ms'.format(data[key]['send_time']) for key in list(data.keys())]},
            {'align': 'right', 'title': 'Latency',
             'values': ['{} ms'.format(data[key]['latency']) for key in list(data.keys())]},
            {'align': 'right', 'title': 'Receive Time',
             'values': ['{} ms'.format(data[key]['receive_time']) for key in list(data.keys())]},
        ]
        return columns_data


class MultitagCumulativeQuantilesTable(MultitagTable):
    title = "Кумулятивные квантили по тэгам"
    quantiles = '99%', '98%', '95%', '90%', '85%', '80%', '75%', '50%'

    def get_data_values(self, tag):
        """
        gets data for non-precise quantiles for cases and different time intervals.
        """
        sql = '''select 
            quantilesExactWeighted(0.5, 0.75, 0.8, 0.85, 0.9, 0.95, 0.98, 0.99)(bin, cnt)
            from loaddb.rt_microsecond_histograms_buffer
            where job_id={job} 
            and job_date = toDate({job_date})
            and tag in ({cases_with_tag})
            '''
        query_params = self.job_obj.basic_query_params.copy()
        query_params.update({
            'cases_with_tag': ','.join("'{}'".format(escape_string(case)) for case in self.job_obj.cases if
                                       tag in case.split(self.delimiter)),
        })
        raw_data = self.ch_client.select(sql, query_params=query_params)
        quantiles = [float(q)/1000 for q in reversed(raw_data[0][0])] \
            if raw_data and raw_data[0] else [0] * len(list(self.rows.keys()))
        return dict(zip(self.quantiles, quantiles))

    def get_columns(self):
        """
        returns list of http codes and it's count
        """
        columns_data = [
            {'weight': 'bold', 'align': 'left', 'title': ' ',
             'values': self.tags},
        ]
        tags_data = {tag: self.get_data_values(tag) for tag in self.tags}
        for quantile in self.quantiles:
            column = [{'align': 'right', 'title': quantile, 'values': [tags_data[tag][quantile] for tag in self.tags]}]
            columns_data += column
        return columns_data


class MultitagHttpCodesTable(MultitagTable):
    title = 'HTTP коды по тэгам'

    def get_httpcode_data(self):
        data = []
        for tag in self.tags:
            sql = '''
                select code, toFloat32(sum(cnt))
                from loaddb.proto_codes_buffer
                where job_id={job} 
                and job_date = toDate({job_date})
                and tag in ({cases_with_tag})
                group by code
            '''
            query_params = self.job_obj.basic_query_params.copy()
            query_params.update({
                'cases_with_tag': ','.join("'{}'".format(escape_string(case)) for case in self.job_obj.cases if
                                           tag in case.split(self.delimiter)),
            })
            fetched_data = self.ch_client.select(sql, query_params=query_params)
            data.extend([tag] + list(d) for d in fetched_data)
        return data

    def get_columns(self):
        """
        returns list of http codes and it's count
        """
        data = self.get_httpcode_data()
        http_codes = sorted(set(c[1] for c in data))
        columns_data = [
            {'weight': 'bold', 'align': 'left', 'title': ' ',
             'values': self.tags},
        ]
        for code in http_codes:
            column = {'align': 'right', 'title': str(code), 'values': []}
            for tag in self.tags:
                tag_val = 0
                i = 0
                while i < len(data):
                    if data[i][1] == code and data[i][0] == tag:
                        tag_val += data[i][2]
                    i += 1
                column['values'].append(tag_val)
            columns_data.append(column)
        return columns_data


class MultitagNetCodesTable(MultitagTable):
    title = 'Сетевые коды по тэгам'

    def get_netcode_data(self):
        data = []
        for tag in self.tags:
            sql = '''select code, toFloat32(sum(cnt))
                    from loaddb.net_codes_buffer
                    where job_id={job} 
                    and job_date = toDate({job_date})
                    and tag in ({cases_with_tag})
                    group by code'''
            query_params = self.job_obj.basic_query_params.copy()
            query_params.update({
                'cases_with_tag': ','.join("'{}'".format(escape_string(case)) for case in self.job_obj.cases if
                                           tag in case.split(self.delimiter)),
            })
            fetched_data = self.ch_client.select(sql, query_params=query_params)
            data.extend([tag] + list(d) for d in fetched_data)
        return data

    def get_columns(self):
        """
        returns list of http codes and it's count
        """
        data = self.get_netcode_data()
        net_codes = sorted(set(c[1] for c in data))
        columns_data = [
            {'weight': 'bold', 'align': 'left', 'title': ' ',
             'values': self.tags},
        ]
        for code in net_codes:
            column = {'align': 'right', 'title': str(code), 'values': []}
            for tag in self.tags:
                tag_val = 0
                i = 0
                while i < len(data):
                    if data[i][1] == code and data[i][0] == tag:
                        tag_val += data[i][2]
                    i += 1
                column['values'].append(tag_val)
            columns_data.append(column)
        return columns_data
