# -*- coding: utf-8 -*-
from api.util.loadscheme import LoadSchemeIterator, PseudoLoadscheme
from common.util.decorators import cached, Memoize
from common.util.clients import ClickhouseClient
from common.util import Ini2Json, coolify, coolify_even_more
from django.contrib.auth.models import User
from django.core.exceptions import ObjectDoesNotExist
from django.db import transaction, models, DatabaseError
from django.db.models.constants import LOOKUP_SEP
from django.db.models.deletion import DO_NOTHING
import datetime
import logging
import re
import time
import uuid
import json
import yaml
from copy import deepcopy
from ConfigParser import Error as ConfigParserError
from BeautifulSoup import BeautifulSoup


class Server(models.Model):
    n = models.AutoField(primary_key=True)
    host = models.CharField(max_length=128)
    is_test = models.PositiveSmallIntegerField(1, default=0)
    fd = models.DateTimeField(default=datetime.datetime.now)
    td = models.DateTimeField(null=1)  # default='0000-00-00 00:00:00')
    config = models.TextField(null=1)
    dsc = models.TextField(null=1)
    last_ip = models.CharField(max_length=20, default='')
    last_dc = models.SmallIntegerField(2, default=0)

    class Meta:
        db_table = 'server'

    @property
    def name(self):
        if self.dsc:
            return self.dsc
        else:
            return self.host


class Project(models.Model):
    n = models.AutoField(primary_key=True)
    owner = models.CharField(max_length=64)
    name = models.CharField(max_length=120, default='default')

    class Meta:
        db_table = 'project'


class JobManager(models.Manager):
    online_kw = {'td__isnull': True}

    def online(self):
        return self.filter(**self.online_kw)


class Job(models.Model):
    n = models.AutoField(primary_key=True)
    project = models.ForeignKey(Project, to_field='n', db_column='project')
    fd = models.DateTimeField(default=datetime.datetime.now)
    td = models.DateTimeField(null=1)  # default='0000-00-00 00:00:00')
    user = models.ForeignKey(User, null=1)
    name = models.CharField(max_length=120, default='')
    dsc = models.TextField(null=1)
    type = models.PositiveSmallIntegerField(3, default='1')
    tank = models.ForeignKey(Server, to_field='n', db_column='tank',
                             related_name='tank_job')  # PositiveIntegerField(10)
    command_line = models.CharField(max_length=512, null=1)
    ammo_path = models.CharField(max_length=256, null=1)
    loop_cnt = models.FloatField(null=1)
    quit_status = models.PositiveSmallIntegerField(1, null=1)
    test_type = models.PositiveSmallIntegerField(1, null=1, default=0)
    srv = models.ForeignKey(Server, to_field='n', db_column='srv',
                            related_name='target_job')  # PositiveIntegerField(10)
    srv_port = models.PositiveIntegerField(8, default=0, null=1)
    instances = models.PositiveIntegerField(8)
    flag = models.PositiveSmallIntegerField(1, default=0)
    ver = models.CharField(max_length=120, null=1)
    configinfo = models.TextField(null=1)
    finalized = models.PositiveSmallIntegerField(1, default=1)
    is_deleted = models.BooleanField(default=False)

    
    class Deleted(ObjectDoesNotExist):
        pass


    @staticmethod
    def check_job(job):
        if not isinstance(job, Job):
            raise Job.DoesNotExist
        elif job.is_deleted or DeletedJob.objects.filter(n=job.n):
            raise Job.Deleted
        return job

    @property
    def id(self):
        """
        n alias
        returns int instead of long
        """
        return int(self.n)

    @property
    def cool_id(self):
        return coolify(self.pk)

    @property
    def cooler_id(self):
        return coolify_even_more(self.pk)

    @property
    def author(self):
        """
        person alias
        """
        return self.person

    @property
    def start(self):
        """
        fd alias
        """
        return self.fd

    @property
    def status(self):
        """
        text representing td presence or absence
        """
        if self.td:
            return 'offline'
        else:
            return 'online'

    @property
    def is_online(self):
        return not bool(self.td)

    @property
    def config(self):
        config_str = self.configinitial or self.configinfo
        try:
            return deepcopy(Ini2Json(config_str).convert())
        except ConfigParserError:
            try:
                config_str = yaml.safe_load(config_str.replace('!!python/unicode ', ''))
                assert isinstance(config_str, dict), 'Invalid config {}'.format(config_str)
                return deepcopy(config_str)
            except (yaml.constructor.ConstructorError, yaml.parser.ParserError, AssertionError) as exc:
                logging.error(repr(exc))
                return {}

    @property
    def config_fmt(self):
        config = self.configinitial or self.configinfo
        try:
            Ini2Json(config).convert()
            return 'ini'
        except ConfigParserError:
            try:
                yaml.safe_load(config.replace('!!python/unicode ', ''))
                return 'yaml'
            except (ValueError, yaml.parser.ParserError):
                return None

    @property
    @Memoize
    def basic_query_params(self):
        @cached('job_%d_basic_query_params' % self.id)
        def params():
            ch_client = ClickhouseClient()
            sql_job_date = '''
                select toUInt32(any(job_date))
                from loaddb.rt_microsecond_details_buffer
                where job_id=%(job)s
            '''
            job_date = ch_client.select(sql_job_date, query_params={'job': self.n})[0][0]
            if not job_date:
                job_date = int(time.mktime(self.fd.timetuple()))

            return {
                'job': self.n,
                'job_date': job_date,
            }
        return params()

    @property
    @Memoize
    def loadschemes(self):
        """
        List of loadscheme objects (or pseudo loadscheme objects if retrieved from configinfo)
        """
        try:
            loadschemes = [ls for ls in LoadScheme.objects.filter(up=self.n,
                                                                  sec_from__lt=(
                                                                      self.data_stopped_unix -
                                                                      self.data_started_unix)
                                                                  ).order_by('sec_from')
                           ]
            assert loadschemes
            return loadschemes
        except AssertionError:
            try:
                ini2json = Ini2Json(self.configinfo)
                configinfo_json = ini2json.convert()
                phantom_section = configinfo_json.get('phantom', {})
                bfg_section = configinfo_json.get('bfg', {})
                schedule = phantom_section.get('rps_schedule', '') \
                    or bfg_section.get('rps_schedule', '') \
                    or phantom_section.get('instances_schedule', '') \
                    or bfg_section.get('instances_schedule', '')
                if re.match(r'%\([a-zA-Z_]+\)s', schedule):
                    schedule = configinfo_json['DEFAULT'][schedule[schedule.index('(') + 1:schedule.index(')')]]
                if schedule.strip():
                    return [PseudoLoadscheme(**ls)
                            for ls in LoadSchemeIterator([l for l in
                                                          schedule.replace(' ', '').replace(')', ') ').split(' ')
                                                          if l],
                                                         self.n)
                            ]
                else:
                    logging.warning('No loadscheme for job %s', self.n)
                    return []
            except:
                logging.exception('')
                return []

    @property
    @Memoize
    def scheme_type(self):
        """
        looks for scheme_type in memcached (rps or instances)
        asks database if there's no data in memcached
        """
        cache_key = 'job_{}_scheme_type'.format(self.n)

        @cached(cache_key, bypass=self.td is None)  # bypass для онлайн стрельб
        def fetch():
            ch_client = ClickhouseClient()
            nonzero_reqps = ch_client.select('''
                select any(job_id)
                from loaddb.rt_microsecond_details_buffer
                where job_id=%(job)s
                and job_date=toDate(%(job_date)s)
                and reqps!=0 
            ''', query_params=self.basic_query_params)
            # any() в новых версиях кликхауса возвращает [[0]], если не находит записей, удовлетворяющих условию
            nonzero_reqps = nonzero_reqps[0][0]
            nonzero_threads = ch_client.select('''
                select any(job_id)
                from loaddb.rt_microsecond_details_buffer
                where job_id=%(job)s
                and job_date=toDate(%(job_date)s)
                and threads!=0
            ''', query_params=self.basic_query_params)
            # any() в новых версиях кликхауса возвращает [[0]], если не находит записей, удовлетворяющих условию
            nonzero_threads = nonzero_threads[0][0]
            if nonzero_reqps:
                scheme_type = 'rps'
            elif not nonzero_reqps and not nonzero_threads:
                # Для случаев ручной заливки пхаута
                scheme_type = 'rps'
            else:
                scheme_type = 'instances'
            return scheme_type

        try:
            return fetch()
        except:
            logging.exception('Could not check reqps for job {} due to:'.format(self.n))
            return 'rps'

    @property
    @Memoize
    def targets(self):
        """
        returns tuple of tuples
        """
        try:
            sql = '''
                select distinct target_host
                from loaddb.monitoring_verbose_data_buffer
                where job_id=%(job)s
                and job_date=toDate(%(job_date)s)
            '''
            ch_client = ClickhouseClient()
            fetched_data = [number[0] for number in ch_client.select(sql, query_params=self.basic_query_params) if
                            number and number[0]]
            targets = [Server.objects.get_or_create(host=host)[0] for host in fetched_data]
            if not targets:
                targets = [self.srv]
            logging.debug("Got targets for job %s quantity of targets :%s", self.n, len(targets))
            return targets
        except:
            logging.exception("Could not get targets for job %s. Taking job srv as target", self.n)
            targets = [self.srv]
            return targets

    @property
    @Memoize
    def cases(self):
        """
        returns a sorted list of cases
        """
        try:
            sql = '''
                select distinct tag 
                from loaddb.rt_microsecond_details_buffer
                where job_id=%(job)s 
                and job_date=toDate(%(job_date)s)
                and tag!='' 
                order by tag'''
            ch_client = ClickhouseClient()
            cases = [case[0].replace('>', '&gt;').replace('<', '&lt;')
                     for case in ch_client.select(sql, query_params=self.basic_query_params)]
        except:
            logging.exception("Could not get job %s cases due to:", self.n)
            cases = []
        return cases

    @property
    @Memoize
    def multitag(self):
        multitag = bool(self.configinfo and '\nmultitag = true\n' in self.configinfo)
        if multitag:
            logging.info('Job %s has multitags', self.n)
        return multitag

    @property
    def tags(self):
        delimiter = '|'
        tags = []
        for case in self.cases:
            tags.extend(case.split(delimiter) if self.multitag else [case])
        return sorted([tag for tag in set(tags)])

    @property
    def monitoring_exists(self):
        """
        checks if monitoring exists for job
        looks for data in monitoring offline data
        """
        ch_client = ClickhouseClient()
        sql = '''
            select any(value)
            from loaddb.monitoring_verbose_data_buffer
            where job_id=%(job)s
            and job_date=toDate(%(job_date)s)
        '''
        exists = ch_client.select(sql, query_params=self.basic_query_params)[0][0]
        logging.info('Monitoring existance check for job %s is: %s', self.n, exists)
        return bool(exists)

    @property
    @Memoize
    def monitoring_only(self):
        """
        for jobs with no data besides monitoring (i.e. mobile stand)
        """
        ch_client = ClickhouseClient()
        data_is_present = bool(ch_client.select('''
            select 
            any(job_id) as j 
            from loaddb.rt_microsecond_details_buffer
            where job_id=%(job)s 
            and job_date=toDate(%(job_date)s)
        ''', query_params=self.basic_query_params)[0][0])
        monitoring_is_present = bool(ch_client.select('''
            select
            any(job_id) as j 
            from loaddb.monitoring_verbose_data_buffer
            where job_id=%(job)s
            and job_date=toDate(%(job_date)s)
        ''', query_params=self.basic_query_params)[0][0])

        return bool(monitoring_is_present and not data_is_present)

    @property
    def imbalance(self):
        try:
            imbalance = JobImbalance.objects.filter(up=self.n).order_by('rob_isimbalance', '-hum_isimbalance', '-n')[
                        0:1].get()
            if imbalance.hum_imbalance:
                return imbalance.hum_imbalance
            elif imbalance.rob_imbalance:
                return imbalance.rob_imbalance
            else:
                return 0
        except ObjectDoesNotExist:
            if self.td:
                logging.warning('JobImbalance DoesNotExist for offline job %s', self.n)
            return 0
        except:
            logging.exception('problems while processing job imbalance for job %s:', self.n)
            return 0

    @property
    def duration(self):
        """
        returns strings
        self.data_stopped - self.data_started for finished jobs
        last trail or now time - self.data_started for online jobs
        """
        if self.td:
            return str(self.data_stopped_unix - self.data_started_unix + 1)
        else:
            try:
                sql_last_trail = '''
                    select toInt32(max(time)) 
                    from loaddb.rt_microsecond_details_buffer
                    where job_id=%(job)s 
                    and job_date=toDate(%(job_date)s)
                '''
                ch_client = ClickhouseClient()
                last_trail_time = ch_client.select(sql_last_trail, query_params=self.basic_query_params)
                last_trail_time = last_trail_time[0][0] if last_trail_time else int(time.time())
                logging.debug('Got Trail for online job %s', self.n)
                return str(last_trail_time - self.data_started_unix)
            except:
                logging.exception('Could not get online Job %s duration due to:', self.n)
                return '0'

    @property
    def estimated_duration(self):
        """
        regarding loadscheme
        """
        try:
            job_loadschemes = LoadScheme.objects.filter(up=self)
            logging.debug("Got %s loadschemes for job %s", len(job_loadschemes), self.n)
            assert job_loadschemes
            job_loadscheme_sec_from = min([loadscheme.sec_from for loadscheme in job_loadschemes])
            job_loadscheme_sec_to = max([loadscheme.sec_to for loadscheme in job_loadschemes])
            return job_loadscheme_sec_to - job_loadscheme_sec_from
        except ObjectDoesNotExist:
            logging.warning('No LoadScheme for online job %s', self.n)
            return 0
        except AssertionError:
            return 0
        except:
            logging.exception('Could not get Job %s estimated duration due to:', self.n)
            return 0

    @property
    def quit_status_text(self):
        """
        converting quit_status codes into text
        """
        job_quit_status_interpretation = {
            '0': 'completed',
            '1': 'interrupted_generic_interrupt',
            '2': 'interrupted',
            '3': 'interrupted_active_task_not_found ',
            '4': 'interrupted_no_ammo_file',
            '5': 'interrupted_address_not_specified',
            '6': 'interrupted_cpu_or_disk_overload',
            '7': 'interrupted_unknown_config_parameter',
            '8': 'interrupted_stop_via_web',
            '9': 'interrupted',
            '11': 'interrupted_job_number_error',
            '12': 'interrupted_phantom_error',
            '13': 'interrupted_job_metainfo_error',
            '14': 'interrupted_target_monitoring_error',
            '15': 'interrupted_target_info_error',
            '21': 'autostop_time',
            '22': 'autostop_http',
            '23': 'autostop_net',
            '24': 'autostop_instances',
            '25': 'autostop_total_time',
            '26': 'autostop_total_http',
            '27': 'autostop_total_net',
            '28': 'autostop_negative_http',
            '29': 'autostop_negative_net',
            '30': 'autostop_http_trend',
            '31': 'autostop_metric_higher',
            '32': 'autostop_metric_lower',
            '100': 'killed_as_stale',
        }
        try:
            if str(self.quit_status) in job_quit_status_interpretation.keys():
                return job_quit_status_interpretation[str(self.quit_status)]
            elif self.quit_status is None:
                return 'online'
            else:
                return 'other'
        except:
            logging.exception('Could not get quit_status for job %s due to:', self.n)
            return 'other'

    @property
    @Memoize
    def data_started(self):
        """
        for online jobs returns first trail or job.fd
        for offline jobs returns JobTrail.trail_start or first Trail.time or job.fd
        """
        try:
            sql_first_trail = '''
                select min(time) 
                from loaddb.rt_microsecond_details_buffer
                where job_id=%(job)s
                and job_date=toDate(%(job_date)s)
            '''
            ch_client = ClickhouseClient()
            first_trail_time = ch_client.select(sql_first_trail, query_params=self.basic_query_params)[0][0]
            return datetime.datetime.strptime(first_trail_time, "%Y-%m-%d %H:%M:%S")
        except IndexError:
            logging.warning('No data for offline job %s', self.n)
            return self.fd
        except:
            logging.warning('Could not get Job %s data_started due to:', self.n)
            return self.fd

    @property
    @Memoize
    def data_started_unix(self):
        return time.mktime(datetime.datetime.timetuple(self.data_started))

    @property
    def data_stopped(self):
        """
        for online jobs returns None
        for offline jobs returns JobTrail.trail_stop or last Trail.time or job.td
        """
        try:
            assert self.td
            try:
                sql_last_trail = '''
                    select max(time) 
                    from loaddb.rt_microsecond_details_buffer
                    where job_id=%(job)s
                    and job_date=toDate(%(job_date)s)
                '''
                ch_client = ClickhouseClient()
                last_trail_time = ch_client.select(sql_last_trail, query_params=self.basic_query_params)[0][0]
                return datetime.datetime.strptime(last_trail_time, "%Y-%m-%d %H:%M:%S")
            except IndexError:
                logging.warning('No Trail for offline job %s', self.n)
                return self.td
        except AssertionError:
            logging.warning("requested data_stopped property for online job %s", self.n)
            return None
        except:
            logging.warning('Could not get Job %s data_stopped due to:', self.n)
            return self.td

    @property
    @Memoize
    def data_stopped_unix(self):
        return time.mktime(datetime.datetime.timetuple(self.data_stopped))

    objects = JobManager()

    class Meta:
        db_table = 'job'


class TankUserAgent(models.Model):
    job = models.ForeignKey(Job, on_delete=DO_NOTHING)
    user_agent = models.CharField(max_length=512)


class DeletedJob(models.Model):
    n = models.PositiveIntegerField(primary_key=True)
    fd = models.DateTimeField(default=datetime.datetime.now)
    td = models.DateTimeField(null=1)  # default='0000-00-00 00:00:00')
    user = models.ForeignKey(User, null=1)
    name = models.CharField(max_length=120, null=1, default='')
    dsc = models.TextField(null=1)
    type = models.PositiveSmallIntegerField(3, default='1')
    tank = models.ForeignKey(Server, to_field='n', db_column='tank', related_name='tank')  # PositiveIntegerField(10)
    command_line = models.CharField(max_length=512, null=1)
    ammo_path = models.CharField(max_length=256, null=1)
    loop_cnt = models.FloatField(null=1)
    quit_status = models.PositiveSmallIntegerField(1, null=1)
    test_type = models.PositiveSmallIntegerField(1, null=1, default=0)
    srv = models.ForeignKey(Server, to_field='n', db_column='srv', related_name='srv')  # PositiveIntegerField(10)
    srv_port = models.PositiveIntegerField(8, default=0)
    instances = models.PositiveIntegerField(8)
    flag = models.PositiveSmallIntegerField(1, default=0)
    ver = models.CharField(max_length=120, null=1)
    configinfo = models.TextField(null=1, default='')
    finalized = models.PositiveSmallIntegerField(1, default=1)

    class Meta:
        db_table = 'deleted_job'


class JobImbalance(models.Model):
    n = models.AutoField(primary_key=True)
    up = models.ForeignKey(Job, to_field='n', db_column='up', on_delete=models.CASCADE)
    hum_isimbalance = models.IntegerField(1, default=0)
    hum_imbalance = models.IntegerField(8, default=0)
    rob_isimbalance = models.IntegerField(1, default=0)
    rob_warning_sec = models.IntegerField(8, default=0)
    rob_imbalance = models.IntegerField(8, default=0)
    rob_imbalance_sec = models.IntegerField(8, default=0)
    hum_processed = models.IntegerField(1, default=0)
    user = models.CharField(max_length=64, null=1)

    class Meta:
        db_table = 'job_imbalance'


class JobComment(models.Model):
    job = models.PositiveIntegerField(null=0)
    author = models.CharField(max_length=64, null=1)
    text = models.TextField(null=1)
    reply = models.PositiveSmallIntegerField(1, default=0)
    thread = models.PositiveIntegerField(8, null=0)
    created_at = models.DateTimeField(default=datetime.datetime.now)
    edited_at = models.DateTimeField(null=1)

    class Meta:
        db_table = 'job_comment'


class LoadScheme(models.Model):
    n = models.AutoField(primary_key=True)
    up = models.ForeignKey(Job, to_field='n', db_column='up')
    sec_from = models.PositiveIntegerField(10)
    sec_to = models.PositiveIntegerField(10)
    load_type = models.PositiveIntegerField(2)
    load_from = models.FloatField(2)
    load_to = models.FloatField(2)
    dsc = models.CharField(max_length=128)

    class Meta:
        db_table = 'loadscheme'


class CustomUserReport(models.Model):
    n = models.AutoField(primary_key=True)
    user = models.CharField(max_length=64)
    name = models.CharField(max_length=32)
    active = models.PositiveSmallIntegerField(1, default=1)
    plots_json = models.TextField(null=1)

    @property
    def plots(self):
        try:
            return json.loads(self.plots_json)
        except Exception as exc:
            logging.error('could not parse cur plots from json: %s', self.plots_json)
            return []

    @classmethod
    def plots_order(cls):
        """
        Порядок графиков для отображения.
        :return: list
        """
        try:
            # FIXME: Не завязываться на абсолютный путь
            with open('/usr/lib/overload/www/offlinepage/templates/custom_report_form.html') as f:
                html = f.read()
            parsed_html = BeautifulSoup(html)
            return [dict(element.attrs)['name'] for element in parsed_html.findAll('input', attrs={'type': 'checkbox'})]
        except:
            logging.exception('')
            return []

    class Meta:
        db_table = 'custom_user_report'


class UploadToken(models.Model):
    n = models.AutoField(primary_key=True)
    job = models.BigIntegerField(10, null=True)
    token = models.CharField(max_length=64, default='')

    class Meta:
        db_table = 'upload_token'


class ApiTokenManager(models.Manager):
    def get_or_create(self, **kwargs):
        """
        Looks up an object with the given kwargs, creating one if necessary.
        Returns a tuple of (object, created), where created is a boolean
        specifying whether an object was created.
        """
        defaults = kwargs.pop('defaults', {})
        lookup = kwargs.copy()
        for f in self.model._meta.fields:
            if f.attname in lookup:
                lookup[f.name] = lookup.pop(f.attname)
        try:
            self._for_write = True
            return self.get(**lookup), False
        except self.model.DoesNotExist:
            try:
                params = dict((k, v) for k, v in kwargs.items() if LOOKUP_SEP not in k)
                params.update(defaults)
                # ===============================================================
                # redefined for creating token
                # ===============================================================
                params['token'] = uuid.uuid4().hex
                obj = self.model(**params)
                with transaction.atomic(using=self.db):
                    obj.save(force_insert=True, using=self.db)
                return obj, True
            except DatabaseError:
                logging.exception('')
                return self.get(**lookup), False


class ApiToken(models.Model):
    n = models.AutoField(primary_key=True)
    user = models.ForeignKey(User)
    token = models.CharField(max_length=64, default='')
    created = models.DateTimeField(default=datetime.datetime.now)
    expires = models.DateTimeField(null=True)
    approved = models.PositiveSmallIntegerField(1, default=1)

    objects = ApiTokenManager()

    class Meta:
        db_table = 'user_api_token'


class UISettings(models.Model):
    user = models.ForeignKey(User, null=1)
    param = models.CharField(max_length=256)
    onoff = models.PositiveSmallIntegerField(1, default=1)
