from json import JSONDecodeError

from common.models import Data, DataMeta, JobMeta, Job, Case, Regression, JobStatusChoice, DataTypeOrder, Config
from common.util import escape_string, ClickhouseClient, log_time_decorator, ClickhouseException, log_time_context
from django.http import HttpResponse, HttpResponseBadRequest, HttpRequest, QueryDict, HttpResponseNotAllowed
from django.core.exceptions import ObjectDoesNotExist
from django.views.decorators.csrf import csrf_exempt
from django.db.utils import IntegrityError
from datetime import datetime
from urllib.parse import urlencode, quote
from typing import Dict, Set, Tuple, List, Union
import json
from retrying import retry
import sys
import warnings
import zipfile
import tempfile
import os
import io
import shutil
import requests
import logging
import pandas as pd
import asyncio

from regression.services import compute_new_test_in_bg

if not sys.warnoptions:
    warnings.simplefilter("ignore")

logger = logging.getLogger(__name__)

RETRY_ARGS = dict(
    wrap_exception=True,
    stop_max_delay=10000,
    wait_fixed=1000,
    stop_max_attempt_number=5
)


@retry(**RETRY_ARGS)
def send_chunk(session, req, timeout=5):
    r = session.send(req, verify=False, timeout=timeout)
    if r.status_code != 200:
        logging.error('CLICKHOUSE_EXCEPTION: {}\n{}\n'.format(r.status_code, r.content))
        raise ClickhouseException('{}\n{}\n'.format(r.status_code, r.content))
    return r


def _add_job_to_regression(job: 'Job', names_uniq: List[str]) -> List[str]:
    errors = []
    for name in names_uniq:
        name = name.lstrip()
        try:
            regression = Regression.objects.get(name=name)
            job.regression_set.add(regression)
            logger.info('Job {} added to regression {}'.format(job.id, regression.name))
        except ObjectDoesNotExist:
            errors.append('Regression with name {} not found'.format(name))
    return errors


@csrf_exempt
def create_job(request):
    """
    Создает джобу
    Принимает параметры джобы в виде простого тела POST запроса.
    test_start обязателен и попадает в свойства самой джобы.
    На все остальные параметры, переданные так в теле создаются записи в таблице job_meta.
    :param request: django HTTP request
    :return: django HTTP response 200 (id джобы плэйн текстом) | 400
    """
    request_data = request.POST
    try:
        test_start = int(request_data['test_start'])
        host = request_data.get('host')
        port = int(request_data.get('port', 0)) or None
        local_uuid = request_data.get('local_id')
        artifact_configs = json.loads(request_data.get('configs', '{}'))
        assert isinstance(artifact_configs, dict), 'Body should be dict, encoded in json'
        assert all(map(lambda key: isinstance(key, str), artifact_configs.keys())), 'Configs keys should be filenames'
        assert all(map(lambda value: isinstance(value, str), artifact_configs.values())), 'Configs values should be strings'
    except (KeyError, ValueError):
        error = 'Request url should contain host: string, port: integer, uuid: string, test_start: integer, ' \
                'configs: json-encoded dict'
        logger.warning('Create job error:' + error)
        return HttpResponseBadRequest(error)
    except AssertionError as e:
        return HttpResponseBadRequest(repr(e))

    job_id, errors = _update_job_info(job_id=None,
                                      host=host,
                                      port=port,
                                      local_uuid=local_uuid,
                                      test_start=test_start,
                                      meta=request_data,
                                      mode='create')
    if job_id:
        _add_job_configs(job_id, artifact_configs)
        return HttpResponse(job_id)
    else:
        return HttpResponseBadRequest(errors)


def _add_job_configs(job_id, configs):
    job = Job.objects.get(id=job_id)
    created_configs = [Config(job=job, name=name, content=content) for name, content in configs.items()]
    Config.objects.bulk_create(created_configs)


@csrf_exempt
def create_metric(request):
    """
    Создает метрику (data) и мета информацию к ней
    Принимает параметры метрики в виде простого тела POST запроса.
    - job - обзателен, указыает к какой джобе принадлежит эта метрика
    - offset становится параметром метрики (по умолчанию 0)
    - type становится параметром метрики (по умолчанию metrics)
    На все остальные параметры, переданные в теле создаются записи в таблице data_meta.
    name и group желательны, но имеют дефолт в виде --NONAME--
    :param request: django HTTP request
    :return: django HTTP response 200 (id метрики плэйн текстом) | 400
    """
    try:
        request_data = json.loads(request.body.decode('utf-8'))
    except JSONDecodeError:
        request_data = request.POST
    logger.debug('Create metric request:\n{}'.format(request_data))
    try:
        tag = _create_data_entry(request_data['job'],
                                 request_data.get('offset', 0),
                                 request_data.get('type', 'metrics'),
                                 request_data.get('types', [request_data.get('type', 'metrics')]),
                                 request_data.get('case'),
                                 request_data.get('parent'),
                                 request_data.get('meta', {}))
    except (Job.DoesNotExist, CreationError) as e:
        return HttpResponseBadRequest(json.dumps({'error': repr(e)}), content_type='application/json; charset=utf-8')
    return HttpResponse(tag)


class CreationError(Exception):
    pass


def _create_data_entry(job_id, offset, d_type, d_types, case=None, parent=None, meta=None):
    """
    Creates Case entry and Data entry if not exists
    :param job_id: job id
    :param offset: офсет метрики
    :param d_type: тип метрики
    :param list d_types: list of data types, choices from DataTypeChoice
    :return: uuid for data upload to ClickHouse
    """
    meta = {} if meta is None else meta

    if bool(case) != bool(parent):
        raise CreationError('"case" and "parent" should both be None or not None. Empty string is treated as None')
    elif case and case != Case.OVERALL and parent:
        data_obj = Case.objects.get(tag=parent).data
        new_case = Case.objects.create(data=data_obj, name=case)
    elif not case or case == Case.OVERALL and not parent:
        data_obj = Data.objects.create(job=Job.objects.get(pk=job_id),
                                       offset=offset,
                                       type=d_type,
                                       types=[int(i.name in d_types) for i in DataTypeOrder])
        new_case = Case.objects.create(data=data_obj, name=Case.OVERALL)
        for k, v in meta.items():
            if k not in ('job', 'uniq_id', 'offset'):
                DataMeta.objects.create(data=data_obj,
                                        key=k,
                                        value=v)
    else:
        raise CreationError('Case {} cant have parent tag {}'.format(case, parent))
    return new_case.tag.hex


@csrf_exempt
def update_job(request):
    """
    Обновляет свойства стрельбы и мета информацию стрельбы
    Вызывается клиентами для заливки данных
    Принимает параметры в виде простого тела POST запроса или json.
    status и test_start обновляются прямо в записи самой джобы в таблице job
    для остальных параметров обновляется или создается запись в таблице job_meta
    :param request: django HTTP request
    - job - номер изменяемой джобы
    :return: django HTTP response 200 | 400
    """

    errors = []
    request_data = {}
    is_json = False
    job = None

    try:
        job_id = request.GET.get('job')
        job = _get_job_object(job_id)
        if request.content_type.endswith('json'):
            is_json = True
            request_data = json.loads(request.body.decode('utf-8'))
        else:
            for item in request.body.decode('utf-8').split('&'):
                key, value = item.split('=')
                request_data[key] = value
    except ValueError:
        errors.append('Failed to parse request body')
    except AssertionError:
        errors.append('Invalid job id {}'.format(job_id))
    except ObjectDoesNotExist:
        errors.append('Job {} not found'.format(job_id))

    _, new_errors = _update_job_info(job_id=job_id, meta=request_data, mode='update')
    errors.extend(new_errors)

    if job_id and job:
        if is_json:
            return HttpResponse(json.dumps({job_id: job.full_meta}), content_type='application/json; charset=utf-8')
        else:
            return HttpResponse('')
    else:
        return HttpResponseBadRequest(json.dumps(errors), content_type='application/json; charset=utf-8')


@csrf_exempt
def rewrite_job_meta(request):
    """
    Принимает параметры в теле POST-запроса в json.
    Перезаписывает свойства стрельбы (только status и test_start) и метаинформацию стрельбы
    (все, кроме вхождений в job.immutable_meta).
    Если существующий в базе ключ отсутствует в запросе, он будет удален из базы.
    Значение существующего в базе ключа будет перезаписано значением из запроса.
     :param request: django HTTP request
    - job - номер изменяемой джобы
    :return: django HTTP response 200 | 400
    """
    errors = []
    request_data = {}
    job = None

    job_id = request.GET.get('job')
    try:
        job = _get_job_object(job_id)
        request_data = json.loads(request.body.decode('utf-8'))
    except ValueError:
        errors.append('Failed to parse request body')
    except AssertionError:
        errors.append('Invalid job id {}'.format(job_id))
    except ObjectDoesNotExist:
        errors.append('Job {} not found'.format(job_id))

    _, new_errors = _update_job_info(job_id=job_id, meta=request_data, mode='rewrite')
    errors.extend(new_errors)

    if job_id and job:
        return HttpResponse(json.dumps({job_id: job.full_meta}), content_type='application/json; charset=utf-8')
    else:
        return HttpResponseBadRequest(json.dumps(errors), content_type='application/json; charset=utf-8')


@csrf_exempt
def update_metric(request):
    """
    Обновляет свойства метрики и мета информацию метрики
    Принимает параметры метрики в виде простого тела POST запроса или json.
    offset обновляется прямо в записи самой метрики в таблице data
    для остальных параметров обновляется или создается запись в таблице data_meta
    uniq_id изменить невозможно.
    :param request: django HTTP request
    - tag - уникальный айдишник метрики, которую надо изменить
    :return: 200 | 400
    """
    is_json = False
    try:
        tags = request.GET.getlist('tag')
        assert tags, 'invalid metric tag {}'.format(tags)
        tags = map(escape_string, tags)  # SECURITAY!!!
        if request.content_type.endswith('json'):
            is_json = True
            request_data = json.loads(request.body.decode('utf-8'))
        else:
            request_data = request.POST.dict()
    except (ValueError, AssertionError) as exc:
        return HttpResponseBadRequest(repr(exc))

    data_objects = Data.objects.filter(uniq_id__in=tags)
    for data_obj in data_objects:
        data_obj.offset = request_data.get('_offset') or request_data.get('offset') or data_obj.offset
        data_obj.save()

        for k, v in request_data.items():
            if k not in ('offset', 'type', 'uniq_id', '_offset', '_type', '_uniq_id'):
                dm = DataMeta.objects.get_or_create(
                    data=data_obj,
                    key=k,
                )[0]
                dm.value = v
                dm.save()
    if is_json:
        return HttpResponse(json.dumps({d.uniq_id: d.full_meta for d in data_objects}),
                            content_type='application/json; charset=utf-8')
    else:
        return HttpResponse('')


def close_job(request):
    """
    Выставляет status = finished
    триггерит агрегацию (в т.ч. duration)
    :param request: django HTTP request
    - job - номер закрываемой джобы
    :return: django HTTP request 200 | 400
    """
    job_id = request.GET.get('job')
    duration_str = request.GET.get('duration')
    if job_id is None or duration_str is None:
        return HttpResponseBadRequest('job and duration are required')
    try:
        assert job_id.isdigit(), 'invalid job {}'.format(job_id)
        assert duration_str.isdigit(), 'invalid duration {}'.format(duration_str)
        job = Job.objects.get(pk=job_id)
        duration = int(duration_str)
    except AssertionError as exc:
        return HttpResponseBadRequest(repr(exc))
    else:
        job.status = JobStatusChoice.finished.value
        job.duration = duration
        job.save()
        compute_new_test_in_bg(job)
    return HttpResponse('')


def delete_job(request):
    """
    проставляет джобе параметр is_deleted
    :param request: django HTTP request
    - job - номер удаляемой джобы
    :return: django HTTP response 200 | 400
    """
    try:
        jobs = request.GET.getlist('job')
        assert jobs and all([str(job).isdigit() for job in jobs]), 'invalid jobs {}'.format(jobs)
        job_objects = Job.objects.filter(pk__in=jobs)
        assert job_objects.count(), 'no job objects found with ids {}'.format(jobs)
    except AssertionError as exc:
        return HttpResponseBadRequest(repr(exc))
    for job_obj in job_objects:
        job_obj.is_deleted = True
        job_obj.save()
    return HttpResponse('')


def job_meta(request):
    jobs = request.GET.getlist('job')
    jobs = Job.objects.filter(id__in=jobs)
    return HttpResponse(json.dumps({job.id: job.full_meta for job in jobs}),
                        content_type='application/json; charset=utf-8')


def job_fragments(_, job):
    """

    :param _: Http Request
    :param job: job id
    :return:
    """
    try:
        assert job.isdigit()
        job_obj = Job.objects.get(id=job)
    except (AssertionError, Job.DoesNotExist) as exc:
        return HttpResponseBadRequest(repr(exc))
    return HttpResponse(json.dumps(job_obj.fragments),
                        content_type='application/json; charset=utf-8')


def _update_job_info(
        job_id: Union[int, None], mode: str, meta: dict,
        host: str = '', port: int = 0,
        local_uuid: Union[str, None] = None,
        test_start: Union[int, None] = None) -> Tuple[Union[int, None], List[str]]:
    """
    Создает джобу, если ее нет, или обновляет метаданные у существующей.
    :param job_id: Job id
    :param meta: Meta info in dict
    :param mode: one of create, update or rewrite
    :return: dict {job_id: {job_meta}, 'errors': [list of errors]}
    """
    job = None
    errors = []

    if mode == 'create':
        try:
            job = Job.objects.create(
                status='new',
                host=host,
                port=port,
                local_uuid=local_uuid,
                test_start=test_start
            )
            job_id = job.id
        except IntegrityError:
            errors.append('job id must be unique')
    else:
        try:
            job = _get_job_object(job_id)
        except AssertionError:
            errors.append('invalid job {}'.format(job_id))
        except ObjectDoesNotExist:
            errors.append('job {} not found'.format(job_id))

    if not job:
        return None, errors

    errors_on_update_job_object = _update_job_object(job, meta)
    errors.extend(errors_on_update_job_object)

    old_job_meta = job.full_meta
    keys_to_add, keys_to_update, keys_to_delete = _get_key_diff(set(old_job_meta.keys()), set(meta.keys()), mode)
    errors_on_update_meta = _update_job_meta(job, meta, keys_to_add, keys_to_update, keys_to_delete)
    errors.extend(errors_on_update_meta)

    return job_id, errors


def _get_job_object(job: int) -> "Job":
    """
    Получаем объект с джобой.
    :param job: Job id
    :return: Job object if it's found, or
        exceptions:
         DoesNotExist - не найден объект с этим job id
         MultipleObjectsReturned - найдено несколько объектов с этим job id
         AssertionError - некорректный job_id
    """
    assert job and str(job).isdigit()
    return Job.objects.get(id=job)


def _get_key_diff(old_keys: Set[str], new_keys: Set[str], mode: str) -> Tuple[Set[str], Set[str], Set[str]]:
    """
    Определяем, какие ключи меты нужно удалять, какие добавлять, а какие апдейтить
    """
    if mode == 'rewrite':
        delete = old_keys - new_keys
        update = old_keys & new_keys
        add = new_keys - old_keys
    elif mode == 'update':
        delete = set(())
        update = old_keys & new_keys
        add = new_keys - old_keys
    else:
        delete = set(())
        update = set(())
        add = new_keys
    return add, update, delete


def _update_job_meta(job: 'Job', data, add: Set[str], update: Set[str], delete: Set[str]) -> List[str]:
    """
    Обновляем метаданные для джобы
    """
    errors = []
    for key in delete:
        if key not in job.immutable_meta:
            j_meta = JobMeta.objects.get(job=job, key=key)
            j_meta.delete()
        else:
            errors.append('Impossible to delete immutable field {}'.format(key))
    for key in update:
        if key == 'regression':
            if isinstance(data[key], list):
                errors_on_add_regression = _add_job_to_regression(job, data[key])
            else:
                errors_on_add_regression = _add_job_to_regression(job, [data[key]])
            errors.extend(errors_on_add_regression)
        elif key not in job.immutable_meta:
            j_meta = JobMeta.objects.get(job=job, key=key)
            j_meta.value = data[key]
            j_meta.save()
        else:
            errors.append('Impossible to update immutable field {}'.format(key))
    for key in add:
        if key == 'regression':
            if isinstance(data[key], list):
                errors_on_add_regression = _add_job_to_regression(job, data[key])
            else:
                errors_on_add_regression = _add_job_to_regression(job, [data[key]])
            errors.extend(errors_on_add_regression)
        else:
            JobMeta.objects.create(job=job, key=key, value=data[key])
    return errors


def _update_job_object(job_object: 'Job', request_data: Dict) -> List[str]:
    job_object.status = request_data.get('_status') or request_data.get('status') or job_object.status
    job_object.test_start = request_data.get('_test_start') or request_data.get('test_start') or job_object.test_start
    try:
        job_object.save()
        return []
    except Exception:
        return ['impossible to update job {} with new status or test start time'.format(job_object.id)]


@csrf_exempt
@log_time_decorator
def unfold_job(request):
    """
    Принимает zip файл с данными, сохраненными локально на машинке с вольтой.
    :param request:
    :return:
    """
    path_to_extract_to = None
    tmpfile_path = None
    if request.method != 'POST':
        return HttpResponseNotAllowed(['POST'])
    try:
        data = request.body
        assert data, 'no data in request'
        tmpfile, tmpfile_path = tempfile.mkstemp()
        with open(tmpfile, 'wb') as f:
            assert isinstance(data, bytes), 'expecting zipped file'
            f.write(data)
        assert zipfile.is_zipfile(tmpfile_path), 'expecting zipped file'
        with zipfile.ZipFile(tmpfile_path) as zf:
            files_to_extract = zf.namelist()
            path_to_extract_to = '/tmp/job_to_unfold_{}/'.format(tmpfile_path.rstrip('/').split('/')[-1])
            zf.extractall(path=path_to_extract_to, members=files_to_extract)

        meta_file_path = [fp for fp in files_to_extract if fp.endswith('meta.json')]
        assert meta_file_path, 'failed to find meta.json file'
        meta_file_path = path_to_extract_to + meta_file_path[0]

        # VALIDATING META
        with open(meta_file_path) as metrics_meta_file:
            meta = json.loads(metrics_meta_file.read())
            metrics = meta.get('metrics')
            assert metrics, 'failed to find metrics in meta.json file'
            jobmeta = meta.get('job_meta')
            assert jobmeta, 'failed to find job_meta in meta.json file'
            assert 'test_start' in jobmeta and str(jobmeta['test_start'].isdigit()), \
                'test_start in job_meta is mandatory and must be int of microseconds'
            jobmeta['test_start'] = int(jobmeta['test_start'])

        # CREATING JOB
        create_job_request = HttpRequest()
        create_job_request.method = 'POST'
        create_job_request.POST = QueryDict(urlencode(jobmeta))
        create_job_response = create_job(create_job_request)
        assert create_job_response.status_code == 200, 'failed to create job'
        job_id = create_job_response.content

        ioloop = asyncio.get_event_loop()
        tasks = []

        # CREATING METRICS
        key_date = datetime.strftime(datetime.fromtimestamp(jobmeta['test_start'] / 10 ** 6), '%Y-%m-%d')
        for metric_key in metrics.keys():
            metric_data_file_path = [fp for fp in files_to_extract if fp.endswith(metric_key + '.data')]
            assert metric_data_file_path, 'failed to find metric {} data'.format(metric_key)
            metric_data_file_path = path_to_extract_to + metric_data_file_path[0]
            tasks.append(
                ioloop.create_task(
                    _upload_metric(metrics[metric_key], key_date, job_id, metric_data_file_path)
                )
            )

        wait_tasks = asyncio.wait(tasks)
        ioloop.run_until_complete(wait_tasks)
        ioloop.close()

        return HttpResponse('https://volta.yandex-team.ru/tests/{}'.format(job_id))
    except AssertionError as aexc:
        logging.exception('')
        return HttpResponseBadRequest(repr(aexc))
    finally:
        if tmpfile_path and os.path.exists(tmpfile_path):
            os.remove(tmpfile_path)
        if path_to_extract_to and os.path.exists(path_to_extract_to):
            shutil.rmtree(path_to_extract_to)


async def _upload_metric(dataset_description: dict, key_date: int, job_id: int, data_file_path: str):
    """
    Корутина для обработки одного файла с данными метрики или евентов
    :param dataset_description:
    :param key_date:
    :param job_id:
    :param data_file_path:
    :return:
    """
    ch_client = ClickhouseClient()
    metric_meta = dataset_description.get('meta', {})
    metric_meta['job'] = job_id
    create_metric_request = HttpRequest()
    create_metric_request.method = 'post'
    create_metric_request.POST = QueryDict(urlencode(metric_meta))
    create_metric_response = create_metric(create_metric_request)
    assert create_metric_response.status_code == 200, 'failed to create metric'
    metric_uniq_id = create_metric_response.content.decode('utf-8')
    # UPLOADING DATASET
    req = requests.Request(
        'POST', "{api}&query={query}".format(
            api=ch_client.url,
            query=quote('INSERT INTO {table} FORMAT TabSeparated'.format(
                table=dataset_description['type']
            ))
        )
    )
    with open(data_file_path) as f:
        with log_time_context('FROM CSV'):
            df = pd.read_csv(io.StringIO('ts\tvalue\n' + '\n'.join(f.readlines()[1:])), sep='\t')
    with log_time_context('INSERT ROWS'):
        df.insert(0, 'uniq_id', metric_uniq_id)
        df.insert(0, 'key_date', key_date)
    with log_time_context('TO CSV'):
        req.data = df.to_csv(sep='\t', index=False, header=False, chunksize=10000)
    req.headers = ch_client.session.headers
    prepared_req = req.prepare()
    resp = send_chunk(ch_client.session, prepared_req)
    assert resp.status_code == 200, 'failed to upload metric'
