package ru.yandex.wmtools.common.service;

import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.net.URL;
import java.util.LinkedList;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

import javax.xml.transform.Templates;
import javax.xml.transform.Transformer;
import javax.xml.transform.TransformerConfigurationException;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.stream.StreamResult;
import javax.xml.transform.stream.StreamSource;

import com.Ostermiller.util.CircularByteBuffer;
import com.Ostermiller.util.CircularCharBuffer;
import org.jdom.Attribute;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.input.SAXBuilder;
import org.jdom.output.XMLOutputter;
import org.jdom.xpath.XPath;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Required;

import ru.yandex.wmtools.common.data.xmlsearch.AbstractRequest;
import ru.yandex.wmtools.common.data.xmlsearch.HostPathRequest;
import ru.yandex.wmtools.common.data.xmlsearch.IndexCountRequest;
import ru.yandex.wmtools.common.data.xmlsearch.XmlSearchRequest;
import ru.yandex.wmtools.common.error.InternalException;
import ru.yandex.wmtools.common.error.InternalProblem;
import ru.yandex.wmtools.common.error.UserException;
import ru.yandex.wmtools.common.servantlet.AbstractServantlet;
import ru.yandex.wmtools.common.util.URLUtil;

/**
 * Created by IntelliJ IDEA.
 * User: senin
 * Date: 30.11.2007
 * Time: 20:03:49
 */

// TODO: think about external interface for this service
public class IndexInfoService extends AbstractDbService {
    private static final Logger log = LoggerFactory.getLogger(IndexInfoService.class);

    private static final String DOCS_COUNT_XPATH = "/yandexsearch/response/results/grouping/found-docs[@priority=\"strict\"]";
    private static final String GROUPS_COUNT_XPATH = "/yandexsearch/response/results/grouping/found[@priority=\"strict\"]";
    private static final String ERROR_XPATH = "/yandexsearch/response/error/@code";
    private static final String IS_FAKE_XPATH =
            "/yandexsearch/response/results/grouping/group[1]/doc/properties/_IsFake";

    private static final int MAX_QUEUE_SIZE = 500;
    private static final int CACHE_SHIFT_ITEMS = MAX_QUEUE_SIZE / 20;
    private static final String NOTHING_FOUND_ERROR_CODE = "15";

    private final Map<Long, LinkedList<XmlSearchResponse>> cache = new ConcurrentHashMap<Long, LinkedList<XmlSearchResponse>>();
    private final LinkedList<XmlSearchResponse> queue = new LinkedList<XmlSearchResponse>();

    private XmlSearchService xmlSearchService;

    private boolean outputXmlSearchRequest;
    private boolean outputXmlSearchResponce;

    public Document callXMLSearch(XmlSearchRequest request) throws InternalException {
        log.debug("IndexInfoService.callXMLSearch: Start");
        try {
            Document reqDocument = request.getDocument();
            String mode = convertToMode(
                    (Attribute) XPath.selectSingleNode(reqDocument, "/request/groupings/groupby/@mode"));
            int pageNum = convertToPageNum(
                    (Element) XPath.selectSingleNode(reqDocument, "/request/page"), 0);
            int pageSize = convertToPageSize(
                    (Attribute) XPath.selectSingleNode(reqDocument, "/request/groupings/groupby/@groups-on-page"), 25);
            log.debug("IndexInfoService.callXMLSearch: pageNum = " + pageNum + "pageSize = " + pageSize);
            String params = request.getUrlParams();
            long queryHash = request.getQueryHashCode();
            XmlSearchResponse response = search(queryHash, mode, pageNum, pageSize);
            if (response == null) {
                log.debug("IndexInfoService.callXMLSearch: make real request");
                Document reqDocumentClone = (Document) reqDocument.clone();
                setPageNumAndSize(pageNum, pageSize, reqDocumentClone);
                XMLOutputter outputer = new XMLOutputter();
                outputer.setFormat(outputer.getFormat().setEncoding("UTF-8"));
                final String xslFileName = "/xsl/xmlsearch_filter.xsl";
                String requestDocumentString = outputer.outputString(reqDocumentClone);

                if (outputXmlSearchRequest) {
                    log.debug("requestDocumentString : " + requestDocumentString );
                }

                Reader responseReader =
                        xmlSearchService.callXMLSearch(
                                requestDocumentString,
                                request.getUrlParams(),
                                new XSLTXmlSearchResultParser(xslFileName));
                SAXBuilder builder = new SAXBuilder();
                Document resDocument = builder.build(responseReader);

                if (outputXmlSearchResponce) {
                    XMLOutputter o = new XMLOutputter();
                    o.setFormat(o.getFormat().setOmitDeclaration(true));
                    log.debug("IndexInfoService.callXMLSearch: result " + o.outputString(resDocument));
                }

                response = new XmlSearchResponse(params, resDocument);
                save(response);
                log.debug("IndexInfoService.callXMLSearch: done and saved");
            }
            Document outDoc = extractRequested(reqDocument, response.getDocument());
            log.debug("IndexInfoService.callXMLSearch: Done");
            return outDoc;
        } catch (IOException e) {
            throw new InternalException(InternalProblem.PROCESSING_ERROR, "IOException in callXMLSearch! ", e);
        } catch (JDOMException e) {
            throw new InternalException(InternalProblem.PROCESSING_ERROR, "JDOMException in callXMLSearch! ", e);
        } catch (TransformerConfigurationException e) {
            throw new InternalException(InternalProblem.PROCESSING_ERROR, "TransformerConfigurationException in callXMLSearch! ", e);
        } catch (TransformerException e) {
            throw new InternalException(InternalProblem.PROCESSING_ERROR, "TransformerException in callXMLSearch! ", e);
        }
    }

    private Document extractRequested(Document request, Document response) throws JDOMException, TransformerException, IOException {
        log.debug("IndexInfoService.extractRequested: Start");
        response = (Document) response.clone();
        Element originalRequest = response.getRootElement().getChild("original-request");
        if (originalRequest == null) {
            originalRequest = new Element("original-request");
            response.getRootElement().addContent(originalRequest);
        }
        originalRequest.removeChild("request");
        originalRequest.addContent((Element) request.getRootElement().clone());

        String xslFileName = "/xsl/extract.xsl";
        InputStream xslStream = getClass().getResourceAsStream(xslFileName);
        TransformerFactory transformerFactory = TransformerFactory.newInstance();
        Templates stylesheet = transformerFactory.newTemplates(new StreamSource(xslStream));
        Transformer processor = stylesheet.newTransformer();

        CircularCharBuffer iccb = new CircularCharBuffer(CircularCharBuffer.INFINITE_SIZE);
        new XMLOutputter().output(response, iccb.getWriter());
        StreamSource source = new StreamSource(iccb.getReader());
        iccb.getWriter().close();

        CircularByteBuffer ocbb = new CircularByteBuffer(CircularByteBuffer.INFINITE_SIZE);
        StreamResult result = new StreamResult(ocbb.getOutputStream());

        processor.transform(source, result);
        ocbb.getOutputStream().close();

        SAXBuilder builder = new SAXBuilder();
        Document resultDoc = builder.build(ocbb.getInputStream());
        log.debug("IndexInfoService.extractRequested: Done");
        return resultDoc;
    }

    private void setPageNumAndSize(int pageNum, int pageSize, Document reqDocument) throws JDOMException {
        if (pageNum < 3) {
            return;
        }

        int pn = 1;
        int ps = HostPathRequest.getMaxDocNum() / 8;
        while (true) {
            log.debug("Trying to set 'pn' and 'ps'. pn = " + pn + ", ps = " + ps);
            if ((pn + 1) * ps - 1 >= HostPathRequest.getMaxDocNum()) {
                // if we do such query, we will exceed MAX_DOC_NUM limit :-(
                return;
            }

            if ((pageNum * pageSize) < (pn * ps)) {
                return;
            }

            if (((pageNum + 1) * pageSize - 1) > ((pn + 1) * ps - 1)) {
                ps *= 2;
                continue;
            }

            ((Attribute) XPath.selectSingleNode(reqDocument, "/request/groupings/groupby/@groups-on-page")).setValue(Integer.toString(ps));
            ((Element) XPath.selectSingleNode(reqDocument, "/request/page")).setText(Integer.toString(pn));
            log.debug("IndexInfoService.setPageNumAndSize: pageSize " + pageSize + " -->" + ps + ", pageNum " + pageNum + " -->" + pn);
            break;
        }
    }

    class XmlSearchResponse {
        private final String params;
        private final Document response;
        private long hashCode;

        XmlSearchResponse(String params, Document response) throws JDOMException, InternalException {
            this.params = params;
            this.response = response;
            calculateQueryHashCode();
        }

        String getUrlParams() {
            return params;
        }

        Document getDocument() {
            return response;
        }

        long getQueryHashCode() {
            return hashCode;
        }

        private void calculateQueryHashCode() throws JDOMException, InternalException {
            Element queryElement = (Element) XPath.selectSingleNode(response, "/yandexsearch/request/query");
            if ((queryElement == null) || (queryElement.getText() == null)) {
                throw new InternalException(InternalProblem.PROCESSING_ERROR, "Query not found in xmlsearch response! Possibly #link is not available from backend!");
            }
            String queryText = queryElement.getText();
            this.hashCode = AbstractRequest.getQueryHashCode(queryText, params);
        }

        public String toString() {
            XMLOutputter o = new XMLOutputter();
            o.setFormat(o.getFormat().setOmitDeclaration(true));
            return "params:" + params + "; response:" + o.outputString(response);
        }
    }

    private XmlSearchResponse search(long queryHash, String mode, int pageNum, int pageSize) throws JDOMException {
        int first = pageSize * pageNum + 1;
        int last = pageSize * pageNum + pageSize;
        log.debug("IndexInfoCache: search : [" + first + ", " + last + "]");

        LinkedList<XmlSearchResponse> list = cache.get(queryHash);
        if (list == null) {
            return null;
        }

        for (XmlSearchResponse r : list) {
            Document resDocument = r.getDocument();
            String rMode = convertToMode((Attribute) XPath.selectSingleNode(resDocument, "/yandexsearch/response/results/grouping/@mode"));
            if (!rMode.equals(mode)) {
                continue;
            }
            int rPageNum = convertToPageNum((Element) XPath.selectSingleNode(resDocument, "/yandexsearch/response/results/grouping/page"), 0);
            int rPageSize = convertToPageSize((Attribute) XPath.selectSingleNode(resDocument, "/yandexsearch/response/results/grouping/@groups-on-page"), 0);
            int rFirst = rPageSize * rPageNum + 1;
            int rLast = rPageSize * rPageNum + rPageSize;
            log.debug("IndexInfoCache: search : check ranges : [" + rFirst + ", " + rLast + "]");
            if ((first >= rFirst) && (last <= rLast)) {
                log.debug("IndexInfoCache: search : check ranges : FOUND!");
                return r;
            }
        }
        return null;
    }

    private int convertToPageNum(Element e, int defaultVlaue) {
        if (e != null) {
            return Integer.parseInt(e.getText());
        }
        log.debug("PageNum not found!");
        return defaultVlaue;
    }

    private int convertToPageSize(Attribute attribute, int defaultVlaue) {
        if (attribute != null) {
            return Integer.parseInt(attribute.getValue());
        }
        log.debug("PageSize not found!");
        return defaultVlaue;
    }

    private String convertToMode(Attribute attribute) {
        if (attribute != null) {
            return attribute.getValue();
        }
        return AbstractRequest.GROUP_BY_MODE_FLAT;
    }

    public synchronized void save(XmlSearchResponse response) throws JDOMException {
        long key = response.getQueryHashCode();
        LinkedList<XmlSearchResponse> list = cache.get(key);
        if (list == null) {
            list = new LinkedList<XmlSearchResponse>();
            cache.put(key, list);
        }
        list.add(response);
        queue.addLast(response);
        assureCacheSize(false);
        log.debug("IndexInfoCache: " + queue.size());
    }

    private void assureCacheSize(boolean forceRemove) throws JDOMException {
        if ((queue.size() >= MAX_QUEUE_SIZE) || (forceRemove && !queue.isEmpty())) {
            XmlSearchResponse old = queue.removeFirst();
            long key = old.getQueryHashCode();
            LinkedList<XmlSearchResponse> list = cache.get(key);
            log.debug("IndexInfoCache: assureCacheSize : key = " + key);
            list.remove(old);
            if (list.isEmpty()) {
                cache.remove(key);
            }
        }
    }

    public void pushCache() throws JDOMException {
        for (int i = 0; i < CACHE_SHIFT_ITEMS; i++) {
            assureCacheSize(true);
        }
    }

    public Long extractLinksCount(XmlSearchRequest request) throws InternalException, UserException {
        return extractLinksCount(callXMLSearch(request));
    }

    public Long extractLinksCount(Document response) throws InternalException {
        return extractResultsCount(response, DOCS_COUNT_XPATH);
    }

    public Long extractLinkGroupsCount(Document response) throws InternalException {
        return extractResultsCount(response, GROUPS_COUNT_XPATH);
    }

    private Long extractResultsCount(Document response, String xpath) throws InternalException {
        try {
            Element element = (Element) XPath.selectSingleNode(response, xpath);
            if (element == null) {
                Attribute errorCode = (Attribute) XPath.selectSingleNode(response, ERROR_XPATH);
                if (errorCode == null) {
                    return 0l;
                }
                return (NOTHING_FOUND_ERROR_CODE.equals(errorCode.getValue())) ? 0l : null;
            }

            return Long.parseLong(element.getText());
        } catch (NumberFormatException e) {
            throw new InternalException(InternalProblem.PROCESSING_ERROR, "NumberFormatException in executeXmlSearchCount", e);
        } catch (JDOMException e) {
            throw new InternalException(InternalProblem.PROCESSING_ERROR, "JDOMException in executeXmlSearchCount", e);
        }
    }

    public IndexedResult checkIfIndexed(XmlSearchRequest request) throws InternalException, UserException {
        Document response = callXMLSearch(request);

        try {
            Element element = (Element) XPath.selectSingleNode(response, DOCS_COUNT_XPATH);
            if (element == null) {
                Attribute errorCode = (Attribute) XPath.selectSingleNode(response, ERROR_XPATH);
                if (errorCode == null) {
                    return IndexedResult.NOT_INDEXED;
                }
                return ((NOTHING_FOUND_ERROR_CODE.equals(errorCode.getValue())) ? IndexedResult.NOT_INDEXED : null);
            }

            long indexedCount = Long.parseLong(element.getText());
            if (indexedCount == 0) {
                return IndexedResult.NOT_INDEXED;
            }
            element = (Element) XPath.selectSingleNode(response, IS_FAKE_XPATH);
            if (element == null) {
                log.error("tag " + IS_FAKE_XPATH + " not found in xml_search response!");
                return IndexedResult.INDEXED;    // assume document is NOT fake
            }
            int isFake = Integer.parseInt(element.getText());
            return (isFake == 0) ? IndexedResult.INDEXED : IndexedResult.FAKE;
        } catch (NumberFormatException e) {
            throw new InternalException(InternalProblem.PROCESSING_ERROR, "NumberFormatException in executeXmlSearchCount", e);
        } catch (JDOMException e) {
            throw new InternalException(InternalProblem.PROCESSING_ERROR, "JDOMException in executeXmlSearchCount", e);
        }
    }

    public Long getIndexCount(String hostname, String path) throws UserException, InternalException {
        URL url = AbstractServantlet.prepareUrl(hostname + path, true);
        hostname = AbstractServantlet.getHostName(url);
        path = URLUtil.getRelativeUrl(url);
        if (path.equals("/")) {
            path = "*";
        } else {
            //path = path.endsWith("/") ? path + "*" : path + "/*";
            path = path + "*";
        }
        log.debug(hostname + path);
        return extractLinksCount(new IndexCountRequest(hostname, path));
    }

    public enum IndexedResult {
        INDEXED,
        FAKE,
        NOT_INDEXED
    }

    @Required
    public void setXmlSearchService(XmlSearchService xmlSearchService) {
        this.xmlSearchService = xmlSearchService;
    }

    public void setOutputXmlSearchResponce(boolean outputXmlSearchResponce) {
        // TODO: rename this property
        this.outputXmlSearchResponce = outputXmlSearchResponce;
        log.debug("Property set: outputXmlSearchResponce = " + outputXmlSearchResponce);
    }

    public void setOutputXmlSearchRequest(boolean outputXmlSearchRequest) {
        // TODO: rename this property
        this.outputXmlSearchRequest = outputXmlSearchRequest;
        log.debug("Property set: outputXmlSearchRequest = " + outputXmlSearchRequest);
    }
}
