# -*- coding: utf-8 -*-
from __future__ import print_function

import sys

def find_end_bracket(text, start, end):
    #print('find_end_bracket',  text[start:end], start, end)
    counter = 0
    for i in xrange(start, end):
        x = text[i]
        if x == '[':
            counter = counter + 1
        if x == ']':
            counter = counter - 1
            if counter == 0:
                #print('     find_end_bracket returned', i)
                return i
    return -1

#vla1-1767-vla-click-clickdaemon-new-9306.gencfg-c.yandex.net:9306 0.000770s/0.000879s/connect=0.000230s 0/189 succ 200
def parse_proxy(text):
    splitted = text.split(' ')
    result = {}
    result['host'], result['port'] = splitted[0].split(':')

    splitted_time = splitted[1].split('/')
    result['time_headers'] = splitted_time[0]
    result['time_flush'] = splitted_time[1]
    if len(splitted_time) > 2:
        result['time_connect'] = splitted_time[2].split('=')[1]

    if splitted[2] == 'system_error':
       result['result'] = splitted[2] + ' ' + splitted[3]
       result['error'] = True
       result['backend_error'] = True
    elif splitted[3] == 'system_error':
        result['size'] = splitted[2].split('/')[1]
        result['result'] = splitted[3] + ' ' + splitted[4]
        result['error'] = True
    elif splitted[4] == 'system_error':
        result['size'] = splitted[2].split('/')[1]
        result['result'] = splitted[3] + ' ' + splitted[4] + ' ' + splitted[5]
        if result['result'] == 'client system_error EIO':
            result['client_error'] = True
        if result['result'] == 'client system_error EPIPE':
            result['client_error'] = True
        if result['result'] == 'client system_error ECANCELED':
            result['client_error'] = True
        if result['result'] == 'client system_error ECANCELED':
            result['client_error'] = True
        if result['result'] == 'backend system_error ECANCELED':
            result['backend_error'] = True
        if result['result'] == 'backend system_error ECONNRESET':
            result['backend_error'] = True
        if result['result'] == 'backend system_error ETIMEOUT':
            result['backend_error'] = True
        if result['result'] == 'backend system_error EIO':
            result['backend_error'] = True
        if result['result'] == 'backend system_error EPIPE':
            result['backend_error'] = True
        result['error'] = True
    elif len(splitted) > 5 and splitted[5] == 'system_error':
        result['size'] = splitted[2].split('/')[1]
        result['result'] = splitted[3] + ' ' + splitted[4] + ' ' + splitted[5] + ' ' + splitted[6]
        if result['result'] == 'flush failed system_error EIO':
            result['client_error_flush'] = True
            result['client_error'] = True
        result['error'] = True
    else:
        result['size'] = splitted[2].split('/')[1]
        result['result'] = splitted[3]
        result['code'] = splitted[4]
        if result['result'] == 'succ':
            result['error'] = False
        else:
            result['error'] = True
    return result

class Record:
    def parse_url(self, url):
        self.url = []
        current = ''
        delimeters = ['/', '?']
        for x in url:
            if x in delimeters:
                if current:
                    self.url.append(current)
                current = ''
            else:
                current = current + x


    def parse_tree(self, text, start, end):
        #print ('parse_tree', text[start:end], start, end, self.modules)
        delimeter = text.find(' ', start, end)
        if delimeter == -1:
            print('ERROR')
            print(text, start, end)
        module_name = text[start + 1 : delimeter]
        open_bracket = text.find('[', delimeter, end)
        if open_bracket == -1:
            module_payload = text[delimeter + 1 : end - 1]
            self.modules.append([module_name, module_payload.strip()])
        else:
            module_payload = text[delimeter + 1 : open_bracket]
            self.modules.append([module_name, module_payload.strip()])
            while True:
                end_bracket = find_end_bracket(text, open_bracket, end - 1)
                self.parse_tree(text, open_bracket, end_bracket + 1)
                open_bracket = text.find('[', end_bracket, end)
                if open_bracket == -1:
                    break


    def __init__(self, line):
        self.text = line
        splitted = line.split('\t')
        self.init_impl(splitted)

    def __init__(self, workflow, referer, host, work_time, ip_port, query, timestamp):
        splitted = [ip_port, timestamp, query, work_time, referer, host, workflow]
        self.init_impl(splitted)

    def init_impl(self, splitted):
        splitted_client = splitted[0].split(':')
        self.client_port = splitted_client[-1]
        self.client_ip = ':'.join(splitted_client[0:-1])

        splitted_timestamp = splitted[1].split('T')
        self.timestamp_date = splitted_timestamp[0]
        self.timestamp_time = splitted_timestamp[1]
        if self.timestamp_time.endswith('+0300'):
            self.timestamp_time = self.timestamp_time[:-5]

        self.request = splitted[2]
        splitted_request = self.request.split(' ')

        self.request_method = splitted_request[0]
        if self.request_method.startswith('"'):
            self.request_method = self.request_method[1:]

        self.request_protocol = splitted_request[-1]
        if self.request_protocol.endswith('"'):
            self.request_protocol = self.request_protocol[:-1]

        if len(splitted_request) != 3:
            raise RuntimeError(self.request)
        self.request_path = splitted_request[1]
        self.parse_url(self.request_path)


        self.work_time = splitted[3]
        self.referer = splitted[4]
        self.host = splitted[5]
        self.workload = splitted[6]
        if self.workload.endswith('\n'):
            self.workload = self.workload[:-1]

        self.modules = []
        self.parse_tree(self.workload.strip(), 0, len(self.workload))

        self.regexp_path = []
        for x in self.modules:
            if x[0] == 'regexp_path':
                self.regexp_path.append(x[1])
        self.main_regexp_path = 'unknown'
        if len(self.regexp_path) > 0:
            self.main_regexp_path = self.regexp_path[0]

        self.last_proxies = []
        #print(self.last_proxies)
        index = len(self.modules) - 1
        while self.modules[index][0] == 'proxy':
            #print(self.modules[index][1])
            self.last_proxies.append(parse_proxy(self.modules[index][1]))
            index = index - 1;
        self.last_module = self.modules[index]
        #print(self.last_proxies)
        self.last_proxies.reverse()

        self.type = 'normal'

        #print(self.last_proxies)
        self.client_error = False
        self.return_http_code = 'error'
        self.error_type = 'no_error'
        if len(self.last_proxies) > 0:
            self.error = self.last_proxies[-1]['error']
            if self.error:
                if 'client_error' in self.last_proxies[-1]:
                    self.error_type = 'client_error'
                elif 'backend_error' in self.last_proxies[-1] and self.last_proxies[-1]['backend_error']:
                    self.error_type = self.last_proxies[-1]['result']
                else:
                    self.error_type = 'unknown_error'
            else:
                self.error_type = 'no_error'
                self.return_http_code = self.last_proxies[-1]['code']

        elif self.last_module[0] == 'click':
            self.type = 'clck_lib'
            if 'click_lib_answer' in self.modules[-1][1]:
                self.error = False
                self.return_http_code = self.modules[-1][1].split(' ')[1]
            else:
                self.error = True
                self.error_type = 'unknown_error'

        elif self.last_module[0] == 'active_check_reply':
            self.type = 'active_check_reply'
            self.error = False
            self.return_http_code = '200'

        elif self.last_module == ['cutter', 'bad_request']:
            self.error = True
            self.error_type = 'cutter_bad_request'

        elif self.last_module == ['cutter', 'bad_ssl_request']:
            self.error = True
            self.error_type = 'cutter_bad_ssl_request'

        elif '[h100 [cutter [antirobot [sub_antirobot] [sub_search]]]]' in self.workload:
            self.error = True
            self.error_type = 'no_proxy_error'

        elif 'ECANCELED]] [sub_search]]]]' in self.workload:
            self.error = True
            self.error_type = 'antirobot_cancel_error'

        elif self.last_module[0] == 'errordocument':
            self.error = True
            self.return_http_code = self.last_module[1].split(' ')[1]
            self.error_type = 'errordocument'

        elif self.last_module[0] == 'on_error':
            self.error = True
            self.error_type = 'on_error'

        else:
            self.error = True
            self.error_type = 'unknown_error'

        if self.return_http_code == 'error' and not self.error:
            self.error = True
            self.error_type = 'return_http_code_error'


    def print(self):
        print('text', self.text)
        print('client', self.client_ip, ':', self.client_port)
        print('timestamp_date', self.timestamp_date)
        print('timestamp_time', self.timestamp_time)
        print('request', self.request)
        print('request_method', self.request_method)
        print('request_protocol', self.request_protocol)
        print('request_path', self.request_path)
        print('url', self.url)
        print('work_time', self.work_time)
        print('host', self.host)
        print('referer', self.referer)
        print('workload', self.workload)
        print('modules', self.modules)
        print('regexp_path', self.regexp_path)
        print('main_regexp_path', self.main_regexp_path)
        print('last_proxies', self.last_proxies)
        print('error', self.error)
        print('error_type', self.error_type)
        print('last_module', self.last_module)
        print('return_http_code', self.return_http_code)
        print('type', self.type)
        print('')

    def to_dict(self):
        result = {}
        result['text'] = self.text
        result['client'] = self.client_ip, ':', self.client_port
        result['timestamp_date'] = self.timestamp_date
        result['timestamp_time'] = self.timestamp_time
        result['request'] = self.request
        result['request_method'] = self.request_method
        result['request_protocol'] = self.request_protocol
        result['request_path'] = self.request_path
        result['url'] = self.url
        result['work_time'] = self.work_time
        result['host'] = self.host
        result['referer'] = self.referer
        result['workload'] = self.workload
        result['modules'] = self.modules
        result['regexp_path'] = self.regexp_path
        result['main_regexp_path'] = self.main_regexp_path
        result['last_proxies'] = self.last_proxies
        result['error'] = self.error
        result['error_type'] = self.error_type
        result['last_module'] = self.last_module
        result['return_http_code'] = self.return_http_code
        result['type'] = self.type

def yt_process(workflow, referer, host, work_time, ip_port, query, timestamp):
    record = Record(workflow, referer, host, work_time, ip_port, query, timestamp)
    return record.to_dict()


class Stat:
    def __init__(self):
        self.services = {}
        self.codes = {}
        self.errors = {}
        self.client_errors = 0
        self.backend_errors = 0
        self.bad_requests = 0
        self.requests = 0

    def print(self, indent = 0, print_services = False):
        #print(self.services)
        print(' ' * indent, 'client_errors', self.client_errors)
        print(' ' * indent, 'requests', self.requests)
        print(' ' * indent, 'backend_errors', self.backend_errors)
        print(' ' * indent, 'codes', sorted(list(self.codes.iteritems()), key=lambda c: c[1], reverse = True))
        print(' ' * indent, 'errors', self.errors)
        if print_services:
            print(' ' * indent, 'services', sorted(list(self.services.iteritems()), key=lambda c: c[1], reverse = True))


    def register(self, record):
        #print(record.text)
        self.requests = self.requests + 1
        if record.error_type == 'client_error':
            self.client_errors = self.client_errors + 1
        elif record.error:
            self.backend_errors = self.backend_errors + 1
            self.errors[record.error_type] = self.errors.get(record.error_type, 0) + 1
            if record.error_type == 'unknown_error':
                record.print()

        #record.print()
        self.services[record.main_regexp_path] = self.services.get(record.main_regexp_path, 0) + 1
        self.codes[record.return_http_code] = self.codes.get(record.return_http_code, 0) + 1


def main():
    all_stat = Stat()
    service_stat = {}
    type_stat = {
        'active_check_reply' : Stat(),
        'clck_lib': Stat(),
        'normal': Stat()
    }

    """record = Record('1:1\tT\t  \t\t\t\t[a b[c d]]\t\t\\t')
    record.print()
    print()
    record = Record('1:1\tT\t  \t\t\t\t[a b[c d][e f]]\t\t\\t')
    record.print()
    """

    counter = 0
    for line in sys.stdin:
        counter = counter + 1
        if counter % 100000 == 0:
            print('ALL')
            all_stat.print(4, True)
            print('CLCK')
            type_stat['clck_lib'].print(4)
            print('OTHER')
            type_stat['normal'].print(4)
            """
            for x in service_stat:
                print(x)
                service_stat[x].print()
                print()
            """
        record = Record(line)
        all_stat.register(record)
        type_stat[record.type].register(record)

        s = record.main_regexp_path
        if s not in service_stat:
            service_stat[s] = Stat()
        service_stat[s].register(record)

    print('ALL')
    all_stat.print(4, True)
    print()

    for t in type_stat:
       print(t)
       type_stat[t].print(4)
       print()

    for s in service_stat:
       print(s)
       service_stat[s].print(4)
       print()




main()

#SELECT dict{"error"} AS error, record{"error_type"} AS error_type, record{"return_http_code"} AS return_http_code, record{"main_regexp_path"} AS main_regexp_path, record{"request"} AS request, record{"workload"} AS workload FROM
