# -*- coding: utf-8 -*-
"""
Created on Feb 28, 2014

@author: noob
"""
import logging
from collections import OrderedDict
from hashlib import md5  # чтобы хэшировать гильзы, в них может быть всякая лажа, которую не воспринимает js
from http.client import responses
from itertools import chain

from os import strerror

from common.models import Server
from common.util.meta import escape_string, format_job_dates
from common.util.decorators import memoized_property
from common.util.aggregators import Aggregator, MonitoringAggregator, \
    RTDetailsAggregator, ProtoCodesAggregator, NetCodesAggregator, RTHistogramsAggregator
from comparepage.util import check_data_elements_equal
from comparepage.views.plots.plots_base import CompareTable
from monitoring.models import Metric


class MetaInfoCompareTable(CompareTable):
    title = 'Мета-информация'

    def get_data(self):
        data = {
            'row_keys': list(self.rows.keys()),
            'rows': self.rows,
            'columns': self.columns_data,
            'mask': self.get_comparison_mask(),
            'dim': self.dimension,
            'selectors': self.selectors,
            'default_selector': self.default_selector,
        }
        return data

    def get_comparison_mask(self):
        """
        for highlighting rows with difference (used for meta info table)
        """
        mask = OrderedDict()
        for row in list(self.rows.keys()):
            mask[row] = check_data_elements_equal(self.columns_data_nodiff[row])
        return mask

    @property
    def rows(self):
        rows = OrderedDict((
            ('n', 'Номер'),
            ('task', 'Задача'),
            ('person', 'Танкист'),
            ('name', 'Имя'),
            ('dsc', 'Описание'),
            ('tank', 'Танк'),
            ('command_line', 'Команда'),
            ('loadscheme', 'Схема нагрузки'),
            ('ammo_path', 'Патроны'),
            ('loop_cnt', 'Цикл патронов'),
            ('quit_status', 'Код выхода'),
            ('srv', 'Мишень'),
            ('srv_port', 'Порт'),
            ('instances', 'Потоки'),
            ('flag', 'Звезда'),
            ('component', 'Компонент'),
            ('ver', 'Версия'),
        ))

        return rows

    @memoized_property
    def columns_data_nodiff(self):
        columns = [
            ['{}__{} | {}'.format(job_obj.n, job_obj.task, job_obj.name) for job_obj in self.job_objects],
            [job.task for job in self.job_objects],
            [job.person for job in self.job_objects],
            [job.name if job.name else '' for job in self.job_objects],
            [job.dsc if job.dsc else '' for job in self.job_objects],
            [job.tank_reduced if job.tank_reduced else '' for job in self.job_objects],
            [job.command_line if job.command_line else '' for job in self.job_objects],
            [' '.join(ls.dsc for ls in sorted(job.loadschemes, key=lambda s: s.load_from))
             for job in self.job_objects],
            [job.ammo_path if job.ammo_path else '' for job in self.job_objects],
            [str(job.loop_cnt) for job in self.job_objects],
            ['{}: {}'.format(str(job.quit_status), str(job.quit_status_text)) for job in self.job_objects],
            [job.srv_reduced if job.srv_reduced else '' for job in self.job_objects],
            [str(job.srv_port) for job in self.job_objects],
            [str(job.instances) for job in self.job_objects],
            [str(job.flag) for job in self.job_objects],
            [str(job.component) for job in self.job_objects],
            [job.ver if job.ver else '' for job in self.job_objects],
        ]
        return OrderedDict(zip(list(self.rows.keys()), columns))

    @property
    def columns_data(self):
        return self.columns_data_nodiff


class CasesDistCompareTable(CompareTable):
    title = 'Распределение ответов по гильзам'
    selectors = (
        ('count', 'Количество'),
        ('percent', 'Доля'),
        ('expect', 'Вр.ответа'),
        ('connect_time', 'Вр.соединения'),
        ('send_time', 'Вр.отправки'),
        ('latency', 'Вр.отклика'),
        ('receive_time', 'Вр.получения')
    )

    def get_case_avg_time(self, job_obj, case, param):
        """

        :param job_obj:
        :param case:
        :param param: time fraction
        :return:
        """
        try:
            sql = '''select round({param}, 3)
                    from loaddb.rt_microsecond_details_buffer
                    where job_id={job}
                    and tag='{tag}'
                    and time >= toDateTime({start})
                    and time <= toDateTime({end})
                    '''
            query_params = job_obj.basic_query_params.copy()
            query_params.update({
                'param': self.selectors_sql_map[param],
                'tag': case.replace("'", "\\'"),
                'start': self.intervals[job_obj.n]['start'],
                'end': self.intervals[job_obj.n]['end'],
            })
            fetched_data = self.ch_client.select(sql, query_params=query_params)
            return fetched_data[0][0]
        except AssertionError:
            return 0
        except:
            logging.exception('Could not get avg time for {} JOB {} TAG {}, due to:'.format(self.__class__.__name__,
                              job_obj.n, case))
            return 0

    @memoized_property
    def data_values(self):
        data_values = {}
        for job_obj in self.job_objects:
            try:
                if self.selector in ('expect', 'connect_time', 'send_time', 'latency', 'receive_time'):
                    fetched_data = []
                    for case in job_obj.cases + ['']:
                        fetched_data.append([case, self.get_case_avg_time(job_obj, case, self.selector)])
                else:
                    sql_net = '''select tag, toUInt32(sum(resps))
                            from loaddb.rt_microsecond_details_buffer
                            where job_id={job}
                            and job_date=toDate({job_date})
                            and time >= toDateTime({start})
                            and time <= toDateTime({end})
                            group by tag
                            order by tag desc'''
                    query_params = job_obj.basic_query_params.copy()
                    query_params.update({
                        'start': self.intervals[job_obj.n]['start'],
                        'end': self.intervals[job_obj.n]['end']
                    })
                    fetched_data = self.ch_client.select(sql_net, query_params=query_params)
                    if self.selector == 'percent':
                        fetched_data = list(zip((value[0] for value in fetched_data if value[0]), self.count_percentage(
                            [value[1] for value in fetched_data if value[0]])))
                        fetched_data.append(('', 100.0))
                data_values[job_obj.n] = fetched_data
            except:
                logging.exception('Could not get data_values for job {} due to:'
                                  .format(self.__class__.__name__, job_obj.n))
                data_values[job_obj.n] = (),
        return data_values

    @memoized_property
    def rows(self):
        """
        common cases in all jobs
        """
        rows = [case for case in self.common_cases if case]
        rows = OrderedDict(zip(['n'] + [md5(r.encode('utf-8')).hexdigest() for r in rows], [''] + rows))
        rows[md5('').hexdigest()] = "Весь тест"
        return rows

    @memoized_property
    def columns_data_nodiff(self):
        columns_data = [
            ['{}__{} | {}'.format(job_obj.n, job_obj.task, job_obj.name) for job_obj in self.job_objects]]
        for case in list(self.rows.keys())[1:]:
            columns_data.append([sum([value[1] for value in self.data_values[job_obj.n] if
                                      value[1] and md5(value[0].encode('utf-8')).hexdigest() == case])
                                 for job_obj in self.job_objects])
        return OrderedDict(zip(list(self.rows.keys()), columns_data))

    @property
    def selectors_sql_map(self):
        return dict((
                ('expect', 'toFloat64(connect_time_sum + send_time_sum + latency_sum + receive_time_sum)/1000/resps'),
                ('connect_time', 'toFloat64(connect_time_sum)/1000/resps'),
                ('send_time', 'toFloat64(send_time_sum)/1000/resps'),
                ('latency', 'toFloat64(latency_sum)/1000/resps'),
                ('receive_time', 'toFloat64(receive_time_sum)/1000/resps'),
            ))

    @property
    def dimension(self):
        if self.selector == 'percent':
            return '%'
        elif self.selector == 'expect':
            return 'ms'
        else:
            return ''


class CasesCumulativeQuantilesCompareTable(CompareTable):
    title = 'Кумулятивные квантили по гильзам'
    dimension = 'ms'
    selectors = list(zip(('99', '98', '95', '90', '85', '80', '75', '50'),
                    ('99%', '98%', '95%', '90%', '85%', '80%', '75%', '50%')))

    @memoized_property
    def rows(self):
        """
        common cases in all jobs
        """
        rows = [case for case in self.common_cases if case]
        rows = OrderedDict(zip(['n'] + [md5(r.encode('utf-8')).hexdigest() for r in rows], [''] + rows))
        rows[md5('').hexdigest()] = "Весь тест"
        return rows

    def get_cases_values(self, job_obj):
        """
        gets data for non-precise quantiles for cases and different time intervals.
        """
        sql = '''select
                tag,
                quantilesExactWeighted(0.{quantile})(bin, cnt)
                from loaddb.rt_microsecond_histograms_buffer
                where job_id = {job}
                and job_date=toDate({job_date})
                and time >= toDateTime({start})
                and time <= toDateTime({end})
                group by tag
                order by tag'''
        query_params = job_obj.basic_query_params.copy()
        query_params.update({
            'quantile': self.selector,
            'start': self.intervals[job_obj.n]['start'],
            'end': self.intervals[job_obj.n]['end'],
        })
        fetched_data = self.ch_client.select(sql, query_params=query_params)
        return {v[0]: float(v[1][0]) / 1000 for v in fetched_data}

    @memoized_property
    def columns_data_nodiff(self):
        columns_data = [['{}__{} | {}'.format(job_obj.n, job_obj.task, job_obj.name) for job_obj in self.job_objects]]
        data_values = {job_obj.n: self.get_cases_values(job_obj) for job_obj in self.job_objects}
        columns_data.extend([data_values[job_obj.n].get(case, 0) for job_obj in self.job_objects]
                            for case in self.common_cases[1:] + [''])
        return OrderedDict(zip(list(self.rows.keys()), columns_data))


class CasesHttpCodesCompareTable(CompareTable):
    title = 'HTTP коды по гильзам'
    dimension = ''

    @memoized_property
    def rows(self):
        """
        common cases in all jobs
        """
        rows = self.common_cases[1:]
        rows = OrderedDict(zip(['n'] + [md5(r.encode('utf-8')).hexdigest() for r in rows], [''] + rows))
        rows[md5('').hexdigest()] = 'Весь тест'
        return rows

    @memoized_property
    def selectors(self):
        http_codes = []
        for job_obj in self.job_objects:
            try:
                sql = '''select distinct toString(code)
                        from loaddb.proto_codes_buffer
                        where job_id={job}
                        and job_date=toDate({job_date})
                        and time >= toDateTime({start})
                        and time <= toDateTime({end})
                         '''
                query_params = job_obj.basic_query_params.copy()
                query_params.update({
                    'start': self.intervals[job_obj.n]['start'],
                    'end': self.intervals[job_obj.n]['end'],
                })
                fetched_data = self.ch_client.select(sql, query_params=query_params)
                http_codes.extend(code[0] for code in fetched_data)
            except:
                logging.exception('Could not get selectors for {} JOB {} due to:'
                                  .format(self.__class__.__name__, job_obj.n))
        selectors = list(zip(set(http_codes), set(http_codes)))
        return selectors

    def get_cases_values(self, job_obj):
        """
        gets data for non-precise quantiles for cases and different time intervals.
        """
        sql_http = '''select tag, toUInt32(sum(cnt))
                    from loaddb.proto_codes_buffer
                    where job_id={job}
                    and job_date=toDate({job_date})
                    and code={code}
                    and time >= toDateTime({start})
                    and time <= toDateTime({end})
                    group by tag
                    '''
        query_params = job_obj.basic_query_params.copy()
        query_params.update({
            'code': self.selector,
            'start': self.intervals[job_obj.n]['start'],
            'end': self.intervals[job_obj.n]['end'],
        })
        count = self.ch_client.select(sql_http, query_params=query_params)
        return dict(count)

    @memoized_property
    def columns_data_nodiff(self):
        columns_data = [['{}__{} | {}'.format(job_obj.n, job_obj.task, job_obj.name) for job_obj in self.job_objects]]
        data_values = {job_obj.n: self.get_cases_values(job_obj) if self.selectors else {} for job_obj in
                       self.job_objects}
        columns_data.extend([data_values[job_obj.n].get(case, 0) for job_obj in self.job_objects]
                            for case in self.common_cases[1:] + [''])
        return OrderedDict(zip(list(self.rows.keys()), columns_data))


class CasesNetCodesCompareTable(CompareTable):
    title = "Сетевые коды по гильзам"
    dimension = ''

    @memoized_property
    def rows(self):
        """
        common cases in all jobs
        """
        rows = [case for case in self.common_cases if case]
        rows = OrderedDict(zip(['n'] + [md5(r.encode('utf-8')).hexdigest() for r in rows], [''] + rows))
        rows[md5('').hexdigest()] = "Весь тест"
        return rows

    @memoized_property
    def selectors(self):
        net_codes = []
        for job_obj in self.job_objects:
            try:
                sql = '''select distinct toString(code)
                        from loaddb.net_codes_buffer
                        where job_id={job}
                        and job_date=toDate({job_date})
                        and time >= toDateTime({start})
                        and time <= toDateTime({end})
                         '''
                query_params = job_obj.basic_query_params.copy()
                query_params.update({
                    'start': self.intervals[job_obj.n]['start'],
                    'end': self.intervals[job_obj.n]['end'],
                })
                fetched_data = self.ch_client.select(sql, query_params=query_params)
                net_codes.extend(code[0] for code in fetched_data)
            except:
                logging.exception("Could not get selectors for {} JOB {} due to:"
                                  .format(self.__class__.__name__, job_obj.n))
        selectors = list(zip(set(net_codes), set(net_codes)))
        return selectors

    def get_cases_values(self, job_obj):
        """
        gets cases count for job returns dict {case: count};
        """
        sql_net = '''select tag, toUInt32(sum(cnt))
                    from loaddb.net_codes_buffer
                    where job_id={job}
                    and job_date=toDate({job_date})
                    and code={code}
                    and time >= toDateTime({start})
                    and time <= toDateTime({end})
                    group by tag
                    '''
        query_params = job_obj.basic_query_params.copy()
        query_params.update({
            'code': self.selector,
            'start': self.intervals[job_obj.n]['start'],
            'end': self.intervals[job_obj.n]['end'],
        })
        count = self.ch_client.select(sql_net, query_params)
        return dict(count)

    @memoized_property
    def columns_data_nodiff(self):
        columns_data = [['{}__{} | {}'.format(job_obj.n, job_obj.task, job_obj.name) for job_obj in self.job_objects]]
        data_values = {job_obj.n: self.get_cases_values(job_obj) for job_obj in self.job_objects}
        columns_data.extend([data_values[job_obj.n].get(case, 0) for job_obj in self.job_objects]
                            for case in self.common_cases[1:] + [''])
        return OrderedDict(zip(list(self.rows.keys()), columns_data))


class HttpCodesCompareTable(CompareTable):
    title = 'Распределение HTTP кодов'
    selectors = (
        ('count', 'Количество'),
        ('percent', 'Доля'),
    )

    @memoized_property
    def data_values(self):
        data_values = {}
        for job_obj in self.job_objects:
            try:
                aggregator = ProtoCodesAggregator(job_obj, tag=self.case, start=self.intervals[job_obj.n]['start'],
                                                  end=self.intervals[job_obj.n]['end'])
                data = aggregator.get_raw_data()

                if self.selector == 'percent':
                    data = list(zip((value[0] for value in data), aggregator.count_percentage(value[1] for value in data)))
                data_values[job_obj.n] = data
            except:
                logging.exception('Could not get data_values for {} JOB {} due to:'
                                  .format(self.__class__.__name__, job_obj.n))
                data_values[job_obj.n] = (),
        return data_values

    @memoized_property
    def rows(self):
        """
        all http codes present in all jobs
        """
        http_codes = [value[0] for value in chain.from_iterable(list(self.data_values.values()))]
        http_codes = sorted(set(http_codes))
        http_codes_explained = []
        for code in http_codes:
            try:
                http_codes_explained.append('{} - {}'.format(code, responses[code]))
            except KeyError:
                http_codes_explained.append(str(code) + ' - ???')
        return OrderedDict(zip(['n'] + http_codes, ['n'] + http_codes_explained))

    @memoized_property
    def columns_data_nodiff(self):
        columns_data = [
            ['{}__{} | {}'.format(job_obj.n, job_obj.task, job_obj.name)
             for job_obj in self.job_objects]]
        for http_code in list(self.rows.keys())[1:]:
            columns_data.append(
                [sum([value[1] for value in self.data_values[job_obj.n] if value[1] and int(value[0]) == http_code])
                 for job_obj in self.job_objects])
        return OrderedDict(zip(list(self.rows.keys()), columns_data))

    @property
    def dimension(self):
        if self.selector == 'percent':
            return '%'
        else:
            return ''


class NetCodesCompareTable(CompareTable):
    title = 'Распределение сетевых кодов'
    selectors = (
        ('count', 'Количество'),
        ('percent', 'Доля'),
    )

    @memoized_property
    def data_values(self):
        data_values = {}
        for job_obj in self.job_objects:
            try:
                aggregator = NetCodesAggregator(job_obj, tag=self.case, start=self.intervals[job_obj.n]['start'],
                                                end=self.intervals[job_obj.n]['end'])
                data = aggregator.get_raw_data()

                if self.selector == 'percent':
                    data = list(zip((value[0] for value in data), aggregator.count_percentage(value[1] for value in data)))
                data_values[job_obj.n] = data
            except:
                logging.exception('Could not get httpcodes for {} JOB {} due to:'
                                  .format(self.__class__.__name__, job_obj.n))
                data_values[job_obj.n] = (),
        return data_values

    @memoized_property
    def rows(self):
        """
        all net codes present in all jobs
        """
        rows = [value[0] for value in chain.from_iterable(list(self.data_values.values()))]
        rows = sorted(set(rows))
        return OrderedDict(zip(['n'] + rows, ['n'] + ['{} - {}'.format(code, strerror(code)) for code in rows]))

    @memoized_property
    def columns_data_nodiff(self):
        columns_data = [
            ['{}__{} | {}'.format(job_obj.n, job_obj.task, job_obj.name) for job_obj in self.job_objects]]
        for net_code in list(self.rows.keys())[1:]:
            columns_data.append(
                [sum([value[1] for value in self.data_values[job_obj.n] if value[1] and int(value[0]) == net_code])
                 for job_obj in self.job_objects])
        return OrderedDict(zip(list(self.rows.keys()), columns_data))

    @property
    def dimension(self):
        if self.selector == 'percent':
            return '%'
        else:
            return ''


class TimesDistCompareTable(CompareTable):
    title = 'Распределение времен ответа'
    default_selector = 'quantile'
    selectors = (
        ('count', 'Количество'),
        ('percent', 'Доля'),
        ('quantile', 'Квантиль'),
    )

    @memoized_property
    def data_values(self):
        data_values = {}
        for job_obj in self.job_objects:
            try:
                aggregator = RTHistogramsAggregator(job_obj, tag=self.case, start=self.intervals[job_obj.n]['start'],
                                                    end=self.intervals[job_obj.n]['end'])
                data = aggregator.get_raw_data()
                if self.selector == 'percent':
                    data = list(zip((value[0] for value in data), aggregator.count_percentage(value[1] for value in data)))
                elif self.selector == 'quantile':
                    data = list(zip((value[0] for value in data), aggregator.count_quantiles(value[1] for value in data)))
                data_values[job_obj.n] = data
            except:
                logging.exception('Could not get data_values for job {} due to:'.format(job_obj.n))
                data_values[job_obj.n] = (),
        return data_values

    @memoized_property
    def rows(self):
        """
        all net codes present in all jobs
        """
        rows = []
        for value in list(self.data_values.values()):
            rows.extend(row[0] for row in value)
        rows = ['n'] + sorted(set(rows), reverse=True)
        rows = OrderedDict(zip(rows, rows))
        return rows

    @memoized_property
    def columns_data_nodiff(self):
        columns_data = [
            ['{}__{} | {}'.format(job_obj.n, job_obj.task, job_obj.name) for job_obj in self.job_objects]]
        if self.selector == 'quantile':
            quantiles_data = [[]]
            row = len(list(self.rows.keys())) - 1
            while row > 0:
                quantiles_data.append([])
                job = 0
                while job < len(self.job_objects):
                    try:
                        quantiles_data[len(list(self.rows.keys())) - row].append(
                            dict(self.data_values[[j.n for j in self.job_objects][job]])[
                                [key for key in list(self.rows.keys())][row]])
                    except KeyError:
                        if row == len(list(self.rows.keys())) - 1:
                            quantiles_data[len(list(self.rows.keys())) - row].append(0.0)
                        else:
                            quantiles_data[len(list(self.rows.keys())) - row].append(
                                quantiles_data[len(list(self.rows.keys())) - (row + 1)][job])
                    job += 1
                row -= 1
            columns_data.extend(reversed(quantiles_data[1:]))
        else:
            for time_to in list(self.rows.keys())[1:]:
                columns_data.append(
                    [sum([value[1] for value in self.data_values[job_obj.n] if value[1] and value[0] == time_to])
                     for job_obj in self.job_objects])
        return OrderedDict(zip(list(self.rows.keys()), columns_data))

    @property
    def dimension(self):
        if self.selector in ('percent', 'quantile'):
            return '%'
        else:
            return ''


class QuantilesCumulativeCompareTable(CompareTable):
    title = 'Кумулятивные квантили'
    dimension = 'ms'
    rows = OrderedDict(zip(('n', 'q99', 'q98', 'q95', 'q90', 'q85', 'q80', 'q75', 'q50'),
                            ('', '99%', '98%', '95%', '90%', '85%', '80%', '75%', '50%')))

    @property
    def tag(self):
        return self.case

    @memoized_property
    def data_values(self):
        data_values = {}
        for job_obj in self.job_objects:
            query_params = job_obj.basic_query_params.copy()
            sql = '''select
                quantilesExactWeighted(0.5, 0.75, 0.8, 0.85, 0.9, 0.95, 0.98, 0.99)(bin, cnt)
                from loaddb.rt_microsecond_histograms_buffer
                where job_id={job}
                and job_date=toDate({job_date})
                '''
            if job_obj.multitag and self.tag:
                cases_with_tag = ["'{}'".format(escape_string(str(case))) for case in job_obj.cases if
                                  self.tag in case.split('|')]
                sql += 'and tag in ({cases_with_tag}) '
                query_params['cases_with_tag'] = ','.join(cases_with_tag)
            else:
                sql += '''and tag='{tag}'  '''
                query_params['tag'] = self.tag
            if self.intervals[job_obj.n]['start']:
                sql += '''
                    and time >= toDateTime({start})
                    '''
                query_params['start'] = self.intervals[job_obj.n]['start']
            if self.intervals[job_obj.n]['end']:
                sql += '''
                    and time <= toDateTime({end})
                    '''
                query_params['end'] = self.intervals[job_obj.n]['end']

            raw_data = self.ch_client.select(sql, query_params=query_params)
            quantiles = [float(q) / 1000 for q in reversed(raw_data[0][0])] \
                if raw_data and raw_data[0] else [0] * len(list(self.rows.keys()))
            data_values[job_obj.n] = list(zip(list(self.rows.keys())[1:], quantiles))
        return data_values

    @memoized_property
    def columns_data_nodiff(self):
        columns_data = [
            ['{}__{} | {}'.format(job_obj.n, job_obj.task, job_obj.name) for job_obj in self.job_objects]]
        for quantile in list(self.rows.keys())[1:]:
            try:
                columns_data.append([[value[1] for value in self.data_values[job_obj.n] if value[0] == quantile][0]
                                     for job_obj in self.job_objects])
            except IndexError:
                columns_data.append([0 for _ in self.job_objects])
        return OrderedDict(zip(list(self.rows.keys()), columns_data))


class AggregatesCompareTable(CompareTable):
    title = 'Агрегаты'
    rows = OrderedDict((
        ('n', ''),
        ('minimum', 'Минимум'),
        ('average', 'Среднее'),
        ('median', 'Медиана'),
        ('maximum', 'Максимум'),
        ('stddev', 'Стд. отклонение'),
    ))

    @memoized_property
    def data_values(self):
        data_values = {}
        for job_obj in self.job_objects:
            data_values[job_obj.n] = []
            try:
                aggregator = RTDetailsAggregator(job_obj, tag=self.case, start=self.intervals[job_obj.n]['start'],
                                                 end=self.intervals[job_obj.n]['end'])
                values = aggregator.get_param_data(self.selector)
                values = {
                    'average': values[0],
                    'stddev': values[1],
                    'minimum': values[2],
                    'maximum': values[3],
                    'median': values[4],
                }
                for row in list(self.rows.keys())[1:]:
                    data_values[job_obj.n].append((row, float(values[row])))
            except:
                logging.exception('Could not get data_values for {} JOB {} due to:'
                                  .format(self.__class__.__name__, job_obj.n))
                data_values[job_obj.n] = (),
        return data_values

    @memoized_property
    def columns_data_nodiff(self):
        columns_data = [
            ['{}__{} | {}'.format(job_obj.n, job_obj.task, job_obj.name) for job_obj in self.job_objects]]
        for quantile in list(self.rows.keys())[1:]:
            columns_data.append([[value[1] for value in self.data_values[job_obj.n] if value[0] == quantile][0]
                                 for job_obj in self.job_objects])
        return OrderedDict(zip(list(self.rows.keys()), columns_data))

    @memoized_property
    def selectors(self):
        """
        select data type (i.e. count, percent, quantiles)
        """
        selectors_keys = [
            'resps',
            'expect',
            'connect_time',
            'send_time',
            'latency',
            'receive_time',
            'threads',
        ]
        selectors = [
            'Ответы в секунду',
            'Вр.ответа',
            'Вр.соединения',
            'Вр.отправки',
            'Вр.отклика',
            'Вр.получения',
            'Потоки',
        ]

        return list(zip(selectors_keys, selectors))

    @property
    def dimension(self):
        dimensions_map = {
            'resps': '',
            'expect': ' ms',
            'connect_time': ' ms',
            'send_time': ' ms',
            'latency': ' ms',
            'receive_time': ' ms',
            'threads': '',
        }
        return dimensions_map[self.selector]


class MonitoringAggregatesTable(CompareTable):
    title = 'Агрегаты мониторинга'
    rows = OrderedDict((
        ('n', ''),
        ('minimum', 'Минимум'),
        ('average', 'Среднее'),
        ('median', 'Медиана'),
        ('maximum', 'Максимум'),
        ('stddev', 'Стд. отклонение'),
    ))

    @memoized_property
    def target_obj(self):
        if self.target != '-1':
            return Server.objects.get(n=self.target)

    @memoized_property
    def metrics(self):
        """
        returns Metric queryset
        """
        sql = '''
            select distinct metric_name
            from loaddb.monitoring_verbose_data_buffer
            where job_id in ({jobs})
            and job_date in ({job_dates})
            '''
        query_params = {
            'jobs': ','.join(str(job_obj.n) for job_obj in self.job_objects),
            'job_dates': format_job_dates(self.job_objects),
        }
        if self.target != '-1':
            sql += " and target_host='{target}'"
            query_params['target'] = self.target_obj.host
        metric_objects = [Metric.objects.get_or_create(
            code=m[0])[0] for m in self.ch_client.select(sql, query_params=query_params)] \
            if query_params['jobs'] else []
        return metric_objects

    @property
    def selectors(self):
        try:
            selectors = dict([(str(metric.id), metric.code) for metric in sorted(self.metrics, key=lambda m: m.code)])
            return list(selectors.items())
        except:
            logging.exception('Could not get selectors for {} for {} due to:'.format(self.__class__.__name__,
                              [job_obj.n for job_obj in self.job_objects]))
            return []

    @property
    def dimension(self):
        dimensions_map = {
            'Memory': 'MB',
            'Disk': 'bytes',
            'CPU': '%',
        }
        return dimensions_map.get(dict(self.selectors).get(self.selector, '').split('_')[0], '')

    @memoized_property
    def data_values(self):
        data_values = {}
        for job_obj in self.job_objects:
            data_values[job_obj.n] = []
            try:
                aggregator = MonitoringAggregator(job_obj, start=self.intervals[job_obj.n]['start'],
                                                  end=self.intervals[job_obj.n]['end'])
                values = aggregator.get_raw_data(target=self.target, metric=self.selector)
                values = {
                    'average': values[0][2] if values else 0,
                    'stddev': values[0][3] if values else 0,
                    'minimum': values[0][4] if values else 0,
                    'maximum': values[0][5] if values else 0,
                    'median': values[0][6] if values else 0,
                }
                for row in list(self.rows.keys())[1:]:
                    data_values[job_obj.n].append((row, float(values[row])))
            except:
                logging.exception('Could not get data_values for {} JOB {} due to:'
                                  .format(self.__class__.__name__, job_obj.n))
                data_values[job_obj.n] = (),
        return data_values

    @memoized_property
    def columns_data_nodiff(self):
        columns_data = [
            ['{}__{} | {}'.format(job_obj.n, job_obj.task, job_obj.name) for job_obj in self.job_objects]]
        for quantile in list(self.rows.keys())[1:]:
            columns_data.append([[value[1] for value in self.data_values[job_obj.n] if value[0] == quantile][0]
                                 for job_obj in self.job_objects])
        return OrderedDict(zip(list(self.rows.keys()), columns_data))


class MultitagCompareTable(CompareTable):
    """
    Upper class for actual tables
    """
    delimiter = '|'

    @memoized_property
    def tags(self):
        """
        multitags are got from cases where there is a delimiter;
        some responses can have more then 1 tag;
        i.e. case == 'tag1<delimiter>tag2'
        """
        tags = []
        for job_obj in self.job_objects:
            for case in job_obj.cases:
                tags.extend(case.split(self.delimiter))
        return sorted(set(tags))

    @memoized_property
    def rows(self):
        """
        common tags in all jobs
        table rows ids and names
        """
        tags = []
        for value in list(self.data_values.values()):
            tags.extend(row[0] for row in value)
        tags = sorted(set(tags))
        rows = OrderedDict(zip(['n'] + [md5(t.encode('utf-8')).hexdigest() for t in tags], [''] + tags))
        return rows

    @memoized_property
    def columns_data_nodiff(self):
        columns_data = [
            ['{}__{} | {}'.format(job_obj.n, job_obj.task, job_obj.name) for job_obj in self.job_objects]]
        for tag in list(self.rows.values())[1:]:
            columns_data.append(
                [sum([value[1] for value in self.data_values[job_obj.n] if value[1] and value[0] == tag])
                 for job_obj in self.job_objects])
        return OrderedDict(zip(list(self.rows.keys()), columns_data))


class MultitagDistCompareTable(MultitagCompareTable):
    title = 'Распределение ответов по тэгам'
    selectors = (
        ('count', 'Количество'),
        ('expect', 'Вр.ответа'),
        ('connect_time', 'Вр.соединения'),
        ('send_time', 'Вр.отправки'),
        ('latency', 'Вр.отклика'),
        ('receive_time', 'Вр.получения')
    )

    def get_tag_avg_time(self, job_obj, tag, param):
        try:
            cases_with_tag = ["'{}'".format(escape_string(str(case))) for case in job_obj.cases if
                              tag in case.split(self.delimiter)]
            assert cases_with_tag
            sql = '''select round({param}, 3)
                from loaddb.rt_microsecond_details_buffer
                where job_id={job}
                and job_date=toDate({job_date})
                and tag in ({cases_with_tag})
                and time >= toDateTime({start})
                and time <= toDateTime({end})
                '''
            query_params = job_obj.basic_query_params.copy()
            # FIXME: Вилка
            query_params.update({
                'param': self.selectors_sql_map[param],
                'cases_with_tag': ','.join(cases_with_tag),
                'start': self.intervals[job_obj.n]['start'],
                'end': self.intervals[job_obj.n]['end'],
            })
            fetched_data = self.ch_client.select(sql, query_params=query_params)
            return fetched_data[0][0]
        except AssertionError:
            return 0
        except:
            logging.exception('Could not get avg time for {} JOB {} TAG {}, due to:'.format(self.__class__.__name__,
                              job_obj.n, tag))
            return 0

    def get_tag_count(self, job_obj, tag):
        try:
            cases_with_tag = ["'{}'".format(escape_string(str(case)))
                              for case in job_obj.cases if tag in case.split(self.delimiter)]
            assert cases_with_tag
            sql = '''select toUInt32(sum(resps))
                from loaddb.rt_microsecond_details_buffer
                where job_id={job}
                and job_date=toDate({job_date})
                and tag in ({cases_with_tag})
                and time >= toDateTime({start})
                and time <= toDateTime({end})
                '''
            query_params = job_obj.basic_query_params.copy()
            query_params.update({
                'cases_with_tag': ','.join(cases_with_tag),
                'start': self.intervals[job_obj.n]['start'],
                'end': self.intervals[job_obj.n]['end'],
            })
            fetched_data = self.ch_client.select(sql, query_params=query_params)
            return fetched_data[0][0]
        except AssertionError:
            return 0
        except:
            logging.exception(
                'Could not get count for {} JOB {} TAG {}, due to:'.format(self.__class__.__name__, job_obj.n, tag)
            )
            return 0

    @memoized_property
    def data_values(self):
        data_values = {}
        for job_obj in self.job_objects:
            try:
                fetched_data = []
                if self.selector in ('expect', 'connect_time', 'send_time', 'latency', 'receive_time'):
                    for tag in self.tags:
                        fetched_data.append([tag, self.get_tag_avg_time(job_obj, tag, self.selector)])
                else:
                    for tag in self.tags:
                        fetched_data.append([tag, self.get_tag_count(job_obj, tag)])
                data_values[job_obj.n] = fetched_data
            except:
                logging.exception('Could not get data_values for {} JOB {} due to:'
                                  .format(self.__class__.__name__, job_obj.n))
                data_values[job_obj.n] = (),
        return data_values

    @property
    def selectors_sql_map(self):
        # FIXME: Вилка
        return {
            'old': dict((
                ('expect', '(connect_time_sum + send_time_sum + latency_sum + receive_time_sum)/resps'),
                ('connect_time', 'connect_time_sum/resps'),
                ('send_time', 'send_time_sum/resps'),
                ('latency', 'latency_sum/resps'),
                ('receive_time', 'receive_time_sum/resps'),
            )),
            'new': dict((
                ('expect', 'toFloat64(connect_time_sum + send_time_sum + latency_sum + receive_time_sum)/1000/resps'),
                ('connect_time', 'toFloat64(connect_time_sum)/1000/resps'),
                ('send_time', 'toFloat64(send_time_sum)/1000/resps'),
                ('latency', 'toFloat64(latency_sum)/1000/resps'),
                ('receive_time', 'toFloat64(receive_time_sum)/1000/resps'),
            )),
        }

    @property
    def dimension(self):
        if self.selector == 'count':
            return ''
        else:
            return 'ms'


class MultitagCumulativeQuantilesCompareTable(MultitagCompareTable):
    title = "Кумулятивные квантили по тэгам"
    dimension = 'ms'
    selectors = list(zip(('99', '98', '95', '90', '85', '80', '75', '50'),
                    ('99%', '98%', '95%', '90%', '85%', '80%', '75%', '50%')))

    @memoized_property
    def data_values(self):
        data_values = {}
        for job_obj in self.job_objects:
            try:
                fetched_data = []
                for tag in self.tags:
                    fetched_data.append([tag, self.get_tag_value(job_obj, tag)])
                data_values[job_obj.n] = fetched_data
            except:
                logging.exception("Could not get data_values for job {} due to:".format(job_obj.n))
                data_values[job_obj.n] = (),
        return data_values

    def get_tag_value(self, job_obj, tag):
        """
        gets data for non-precise quantiles for cases and different time intervals.
        """
        try:
            cases_with_tag = ["'{}'".format(escape_string(str(case)))
                              for case in job_obj.cases if tag in case.split(self.delimiter)]
            assert cases_with_tag
            sql = '''
                select
                quantilesExactWeighted(0.{quantile})(bin, cnt)
                from loaddb.rt_microsecond_histograms_buffer
                where job_id={job}
                and job_date=toDate({job_date})
                and tag in ({cases_with_tag})
                and time >= toDateTime({start})
                and time <= toDateTime({end})
            '''
            query_params = job_obj.basic_query_params.copy()
            query_params.update({
                'quantile': self.selector,
                'cases_with_tag': ','.join(cases_with_tag),
                'start': self.intervals[job_obj.n]['start'],
                'end': self.intervals[job_obj.n]['end'],
            })
            fetched_data = self.ch_client.select(sql, query_params=query_params)
            quantile = float(fetched_data[0][0][0]) / 1000
            return quantile
        except AssertionError:
            return 0
        except:
            logging.exception('Could not get value for QUANTILE {} {} JOB {} TAG {}, due to:'.format(self.selector,
                              self.__class__.__name__, job_obj.n, tag))
            return 0


class MultitagHttpCodesCompareTable(MultitagCompareTable):
    title = "HTTP коды по тэгам"
    dimension = ''

    @memoized_property
    def selectors(self):
        http_codes = []
        for job_obj in self.job_objects:
            try:
                sql = '''select distinct toString(code)
                    from loaddb.proto_codes_buffer
                    where job_id={job}
                    and job_date=toDate({job_date})
                    and tag = ''
                    and time >= toDateTime({start})
                    and time <= toDateTime({end})
                    order by code'''
                query_params = job_obj.basic_query_params.copy()
                query_params.update({
                    'start': self.intervals[job_obj.n]['start'],
                    'end': self.intervals[job_obj.n]['end'],
                })
                fetched_data = self.ch_client.select(sql, query_params=query_params)
                http_codes.extend(code[0] for code in fetched_data)
            except:
                logging.exception('Could not get selectors for {} JOB {} due to:'
                                  .format(self.__class__.__name__, job_obj.n))
        selectors = list(zip(set(http_codes), set(http_codes)))
        return selectors

    @memoized_property
    def data_values(self):
        data_values = {}
        for job_obj in self.job_objects:
            try:
                fetched_data = []
                for tag in self.tags:
                    fetched_data.append([tag, self.get_tag_value(job_obj, tag)])
                data_values[job_obj.n] = fetched_data
            except:
                logging.exception("Could not get data_values for {} JOB {} due to:"
                                  .format(self.__class__.__name__, job_obj.n))
                data_values[job_obj.n] = (),
        return data_values

    def get_tag_value(self, job_obj, tag):
        """
        gets data for non-precise quantiles for cases and different time intervals.
        """
        try:
            cases_with_tag = ["'{}'".format(escape_string(str(case))) for case in job_obj.cases if
                              tag in case.split(self.delimiter)]
            assert cases_with_tag
            sql_http = '''select toUInt32(sum(cnt))
                from loaddb.proto_codes_buffer
                where job_id={job}
                and job_date=toDate({job_date})
                and tag in ({cases_with_tag})
                and code = {code}
                and time >= toDateTime({start})
                and time <= toDateTime({end})  '''
            query_params = job_obj.basic_query_params.copy()
            query_params.update({
                'cases_with_tag': ','.join(cases_with_tag),
                'code': self.selector,
                'start': self.intervals[job_obj.n]['start'],
                'end': self.intervals[job_obj.n]['end'],
            })
            fetched_data = self.ch_client.select(sql_http, query_params=query_params)
            assert fetched_data
            value = fetched_data[0][0]
        except AssertionError:
            value = 0
        except:
            logging.exception('')
            value = 0
        return value


class MultitagNetCodesCompareTable(MultitagCompareTable):
    title = "Сетевые коды по тэгам"
    dimension = ''

    @memoized_property
    def selectors(self):
        net_codes = []
        for job_obj in self.job_objects:
            try:
                sql = '''select distinct toString(code)
                    from loaddb.net_codes_buffer
                    where job_id={job}
                    and job_date=toDate({job_date})
                    and tag = ''
                    and time >= toDateTime({start})
                    and time <= toDateTime({end})
                    order by code
                    '''
                query_params = job_obj.basic_query_params.copy()
                query_params.update({
                    'start': self.intervals[job_obj.n]['start'],
                    'end': self.intervals[job_obj.n]['end'],
                })
                fetched_data = self.ch_client.select(sql, query_params=query_params)
                net_codes.extend(code[0] for code in fetched_data)
            except:
                logging.exception('Could not get selectors for {} JOB {} due to:'
                                  .format(self.__class__.__name__, job_obj.n))

        selectors = list(zip(sorted(set(net_codes)), sorted(set(net_codes))))
        return selectors

    @memoized_property
    def data_values(self):
        data_values = {}
        for job_obj in self.job_objects:
            try:
                fetched_data = []
                for tag in self.tags:
                    fetched_data.append([tag, self.get_tag_value(job_obj, tag)])
                data_values[job_obj.n] = fetched_data
            except:
                logging.exception('Could not get data_values for {} JOB {} due to:'
                                  .format(self.__class__.__name__, job_obj.n))
                data_values[job_obj.n] = (),
        return data_values

    def get_tag_value(self, job_obj, tag):
        try:
            cases_with_tag = ["'{}'".format(escape_string(str(case))) for case in job_obj.cases if
                              tag in case.split(self.delimiter)]
            assert cases_with_tag
            sql_net = '''select toUInt32(sum(cnt))
                from loaddb.net_codes_buffer
                where job_id={job}
                and job_date=toDate({job_date})
                and tag in ({cases_with_tag})
                and code = {code}
                and time >= toDateTime({start})
                and time <= toDateTime({end})  '''
            query_params = job_obj.basic_query_params.copy()
            query_params.update({
                'cases_with_tag': ','.join(cases_with_tag),
                'code': self.selector,
                'start': self.intervals[job_obj.n]['start'],
                'end': self.intervals[job_obj.n]['end'],
            })
            fetched_data = self.ch_client.select(sql_net, query_params=query_params)
            assert fetched_data
            value = fetched_data[0][0]
        except AssertionError:
            logging.exception('')
            value = 0
        except:
            logging.exception('')
            value = 0
        return value


class MultitagTimesCompareTable(MultitagCompareTable):
    title = 'Распределение времен по тэгам'
    dimension = '%'

    @memoized_property
    def selectors(self):
        times = []
        for job_obj in self.job_objects:
            try:
                sql = '''select distinct toString(toFloat32(bin)/1000)
                    from loaddb.rt_microsecond_histograms_buffer
                    where job_id={job}
                    and job_date=toDate({job_date})
                    and tag = ''
                    and time >= toDateTime({start})
                    and time <= toDateTime({end})
                    order by bin'''
                query_params = job_obj.basic_query_params.copy()
                query_params.update({
                    'start': self.intervals[job_obj.n]['start'],
                    'end': self.intervals[job_obj.n]['end'],
                })
                fetched_data = self.ch_client.select(sql, query_params=query_params)
                times.extend(code[0] for code in fetched_data)
            except:
                logging.exception('Could not get selectors for {} JOB {} due to:'
                                  .format(self.__class__.__name__, job_obj.n))

        selectors = list(zip(sorted(set(times), key=lambda b: float(b)), sorted(set(times), key=lambda b: float(b))))
        return selectors

    @memoized_property
    def data_values(self):
        data_values = {}
        for job_obj in self.job_objects:
            try:
                fetched_data = []
                for tag in self.tags:
                    fetched_data.append([tag, self.get_tag_value(job_obj, tag)])
                data_values[job_obj.n] = fetched_data
            except:
                logging.exception('Could not get data_values for {} JOB {} due to:'
                                  .format(self.__class__.__name__, job_obj.n))
                data_values[job_obj.n] = (),
        return data_values

    def get_tag_value(self, job_obj, tag):
        try:
            cases_with_tag = ["'{}'".format(escape_string(str(case))) for case in job_obj.cases if
                              tag in case.split(self.delimiter)]
            assert cases_with_tag
            sql = '''select toUInt32(sum(cnt)), bin/1000
                from loaddb.rt_microsecond_histograms_buffer
                where job_id={job}
                and job_date=toDate({job_date})
                and tag in ({cases_with_tag})
                and time >= toDateTime({start})
                and time <= toDateTime({end})
                group by bin
                order by bin desc;'''
            query_params = job_obj.basic_query_params.copy()
            query_params.update({
                'cases_with_tag': ','.join(cases_with_tag),
                'start': self.intervals[job_obj.n]['start'],
                'end': self.intervals[job_obj.n]['end'],
            })
            fetched_data = self.ch_client.select(sql, query_params=query_params)
            assert fetched_data
            times = [int(value[1]) for value in fetched_data]
            data_values = [float(value[0]) for value in fetched_data]
            aggregator = Aggregator()
            data_values = aggregator.count_quantiles(data_values[::-1])[::-1]
            data_values_dict = dict(zip(times, data_values))
            i = len(data_values) - 1
            percentile = 0
            while i >= 0:
                selector_percentile = data_values_dict[max([t for t in times if t <= int(self.selector)])]
                if data_values_dict[times[i]] >= selector_percentile:
                    percentile = data_values[i]
                    break
                else:
                    i -= 1
        except ValueError:
            # no times for this tag with this selector;
            percentile = 0
        except AssertionError:
            percentile = 0
        except:
            logging.exception('Could not get value for TIME {} {} JOB {} TAG {}, due to:'.format(self.selector,
                              self.__class__.__name__, job_obj.n, tag))
            percentile = 0
        return percentile
