#!/usr/bin/env python
# -*- coding: utf-8 -*-

import re
import sys
import os
from optparse import OptionParser
import socket
import logging
import multiprocessing as mp
import gzip
import Queue
import signal
import time

import utils
from errors import IndexerException, IncorrectDocumentException, \
    ClosedConnection

FILES_RE = re.compile(r"^(logstore-.*_)?msearch-proxy_access\.log.*$")

LOG = logging.getLogger('suggest')


def get_filenames(dirname, skipstart, skipend):
    all_files_in_dir = os.listdir(dirname)

    needed_files = []
    for f in all_files_in_dir:
        if FILES_RE.match(f):
            needed_files.append(f)

    needed_files = sorted(needed_files, key=None, reverse=False)
    LOG.debug('%s files in dir, %s needed files in dir %s',
        len(all_files_in_dir), len(needed_files), dirname)

    if skipstart is not None:
        try:
            start_index = needed_files.index(skipstart)
        except ValueError:
            raise RuntimeError("%s not in dirname %s. This should be filename like httpd_msearch-proxy_access.log." % (skipstart, dirname))
    else:
        start_index = 0

    if skipend is not None:
        try:
            end_index = needed_files.index(skipend) + 1
        except ValueError:
            raise RuntimeError("%s not in dirname %s. This should be filename like httpd_msearch-proxy_access.log." % (skipend, dirname))
    else:
        end_index = len(needed_files)

    if start_index > end_index:
        LOG.debug('all files in dir %s indexed', dirname)
        return []

    needed_files = needed_files[start_index:end_index]
    LOG.debug('process %s files', len(needed_files))
    return needed_files


class Getter(object):
    def __init__(self, log_queue, dirname, queue, files_queue):
        self.log_queue = log_queue
        self.dirname = dirname
        self.queue = queue
        self.files_queue = files_queue

    def run(self):
        utils.setup_log_handler(self.log_queue, logging.DEBUG)

        try:
            while True:
                try:
                    filename = self.files_queue.get(block=False)
                    LOG.debug('start getting data from file %s', filename)

                    file_obj = gzip.open(self.dirname + filename, 'r')

                    to_send = 0
                    for suggest_message in utils.parse_log(file_obj):
                        try:
                            self.queue.put(suggest_message)
                            to_send += 1
                        except Exception, e:
                            LOG.warn('failed to create valid proto message for %s: %s', suggest_message, e, exc_info=sys.exc_info())

                    LOG.debug('count line to send %s', to_send)
                    LOG.debug('end getting data from file %s', filename)
                except Queue.Empty:
                    LOG.debug('Done get all data.')
                    self.queue.put('DONE')
                    break
        except Exception, e:
            LOG.error('Some error in getter %s', e, exc_info=sys.exc_info())


class Sender(object):
    def __init__(self, log_queue, queue, hosts, timeout):
        self.log_queue = log_queue
        self.queue = queue

        self.host = hosts
        self.timeout = timeout
        self.socks = {}

    def run(self):
        utils.setup_log_handler(self.log_queue, logging.DEBUG)

        try:
            while True:
                suggest_message = self.queue.get()

                if suggest_message == 'DONE':
                    LOG.debug('Done send all data.')
                    for _, sock in self.socks.items():
                        if sock:
                            sock.close()
                    break

                for h in hosts:
                    send = True
                    while send:
                        try:
                            sock = self.socks.get(h)

                            if sock is None:
                                sock = utils.get_sock(h[0], h[1], self.timeout)
                                self.socks.update({h: sock})

                            utils.send_message(sock, suggest_message, host=h)
                            send = False
                        except socket.timeout:
                            LOG.warn('timeout in socket where indexing message %s',
                                (suggest_message.keyprefix, suggest_message.url), exc_info=sys.exc_info())
                            if sock:
                                sock.close()
                            self.socks.update({h: None})
                        except (socket.error, socket.gaierror), e:
                            LOG.warn('error in socket, where send/get data %s: %s',
                                (suggest_message.keyprefix, suggest_message.url), e, exc_info=sys.exc_info())
                            if sock:
                                sock.close()
                            self.socks.update({h: None})
                        except ClosedConnection, e:
                            LOG.warn('connection is closed %s: %s',
                                (suggest_message.keyprefix, suggest_message.url), e, exc_info=sys.exc_info())
                            if sock:
                                sock.close()
                            self.socks.update({h: None})
                        except IncorrectDocumentException, e:
                            LOG.error('incorrect message %s: %s',
                                (suggest_message.keyprefix, suggest_message.url), e, exc_info=sys.exc_info())
                            send = False
                        except IndexerException, e:
                            LOG.warn('exception where indexing message %s: %s',
                                (suggest_message.keyprefix, suggest_message.url), e, exc_info=sys.exc_info())
                        except Exception, e:
                            LOG.error('some exception in message %s: %s',
                                (suggest_message.keyprefix, suggest_message.url), e, exc_info=sys.exc_info())
                            if sock:
                                sock.close()
                            self.socks.update({h: None})
                            send = False
        except Exception, e:
            LOG.error('Some error in sender %s', e, exc_info=sys.exc_info())


def main(process_num, hosts, timeout, dirname, logname, skipstart=None, skipend=None):
    logger_manager = mp.Manager()
    queues_manager = mp.Manager()
    ### log listener-writer configure
    log_queue = logger_manager.Queue(100000)
    qlistener = utils.setup_log_listener_writer(log_queue, {'log': logname})
    qlistener.start()
    ###
    utils.setup_log_handler(log_queue, logging.DEBUG)

    filenames = get_filenames(dirname, skipstart, skipend)
    filenames_queue = queues_manager.Queue()
    for f in filenames:
        filenames_queue.put(f)

    queues = [queues_manager.Queue(100000) for i in range(process_num)]

    getters = [Getter(log_queue, dirname, queue, filenames_queue)
        for queue in queues]

    getter_procs = [mp.Process(target=getter.run) for getter in getters]

    for getter_proc in getter_procs:
        getter_proc.start()

    senders = [Sender(log_queue, queue, hosts, timeout)
        for queue in queues]

    sender_procs = [mp.Process(target=sender.run) for sender in senders]

    for sender_proc in sender_procs:
        sender_proc.start()

    childs = getter_procs + sender_procs

    # forward termination to childs
    def hook(signum, frame):
        LOG.info('recieved signal %s, terminating childs', signum)
        [proc.terminate() for proc in childs]

    signal.signal(signal.SIGINT, hook)
    signal.signal(signal.SIGCHLD, signal.SIG_IGN)
    signal.signal(signal.SIGTERM, hook)
    signal.signal(signal.SIGQUIT, hook)

    # wait for childs to stop execution
    for p in childs:
        try:
            p.join()
        except:
            continue

    LOG.debug('send reopen_indexers')
    time.sleep(3 * 60)
    for host, port in hosts:
        sock = utils.get_sock(host, port, timeout)
        utils.send_reopen_indexers(sock)
        sock.close()

    LOG.debug('done index')
    qlistener.stop()
    try:
        lm = logger_manager.shutdown()
    except OSError:
        pass
    try:
        qm = queues_manager.shutdown()
    except OSError:
        pass


if __name__ == "__main__":
    parser = OptionParser()

    parser.add_option("-d", "--dirname", dest="dirname",
        help="absolute directory name with logs, e.g. /var/log/")

    parser.add_option("-o", "--processnum", dest="process_num", default=3,
        type="int", help="""At each number of process is created getter process
        and sender process.""")

    parser.add_option("-i", "--hosts", dest="hosts",
        help="list of hosts with port, like localhost:15104,localhost:15105")

    parser.add_option("-t", "--timeout", dest="timeout", type="int",
        help="timeout for indexing requests")

    parser.add_option("-s", "--skipstart", dest="skipstart",
        help="skip already processed files before this filename,"
        " this filename will not be skipped")

    parser.add_option("-e", "--skipend", dest="skipend",
        help="skip already processed files after this filename,"
        " this filename will not be skipped")

    parser.add_option("-l", "--logname", dest="logname",
        help="absolute address to logname filename")

    options, args = parser.parse_args()

    if not options.dirname:
        parser.error("option -d required")

    if not options.process_num:
        parser.error("option -o required")

    if not options.hosts:
        parser.error("option -i required")

    options.hosts = options.hosts.strip()
    hosts = []
    if options.hosts:
        for i in options.hosts.split(','):
            host, port = i.split(':')
            hosts.append((host, int(port)))

    if len(hosts) == 0:
        raise RuntimeError('Empty hosts=%s' % hosts)

    if not options.timeout:
        parser.error("option -t required")

    if not options.logname:
        parser.error("option -l required")

    if options.skipstart is not None:
        skipstart = options.skipstart
    else:
        skipstart = None

    if options.skipend is not None:
        skipend = options.skipend
    else:
        skipend = None

    main(
        options.process_num,
        hosts,
        options.timeout,
        options.dirname,
        options.logname,
        skipstart,
        skipend,
    )
