package ru.yandex.webmaster.common.sitemap;

import java.net.URL;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.*;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Required;
import org.springframework.jdbc.core.RowCallbackHandler;
import org.springframework.jdbc.core.simple.ParameterizedRowMapper;
import org.springframework.transaction.TransactionStatus;

import ru.yandex.common.util.db.LongRowMapper;
import ru.yandex.wmconsole.data.SeverityEnum;
import ru.yandex.wmconsole.data.WMCHistoryObjectTypeEnum;
import ru.yandex.wmconsole.data.info.BriefHostInfo;
import ru.yandex.wmconsole.data.info.HostDbHostInfo;
import ru.yandex.wmconsole.data.info.MergedSitemapInfo;
import ru.yandex.wmconsole.data.info.SitemapInfo;
import ru.yandex.wmconsole.data.info.sitemap.SitemapIndexInfo;
import ru.yandex.wmconsole.data.partition.WMCPartition;
import ru.yandex.wmconsole.data.sitemaps.SitemapFormatEnum;
import ru.yandex.wmconsole.data.sitemaps.SitemapSourceEnum;
import ru.yandex.wmconsole.data.sitemaps.SitemapTypeEnum;
import ru.yandex.wmconsole.service.HostDbHostInfoService;
import ru.yandex.wmconsole.service.error.WMCUserProblem;
import ru.yandex.wmtools.common.data.HistoryActionEnum;
import ru.yandex.wmtools.common.error.InternalException;
import ru.yandex.wmtools.common.error.UserException;
import ru.yandex.wmtools.common.servantlet.AbstractServantlet;
import ru.yandex.wmtools.common.service.AbstractDbService;
import ru.yandex.wmtools.common.service.HistoryService;
import ru.yandex.wmtools.common.util.ServiceTransactionCallback;
import ru.yandex.wmtools.common.util.SqlUtil;
import ru.yandex.wmtools.common.util.URLUtil;

/**
 * This service reads/writes information about sitemaps from/in a database.
 *
 * @author ailyin
 */
public class SitemapService extends AbstractDbService {
    private static final Logger log = LoggerFactory.getLogger(SitemapService.class);

    private static final String FIELD_SITEMAP_ID = "sitemap_id";
    private static final String FIELD_INDEX_ID = "index_id";
    private static final String FIELD_INDEX_ID_COUNT = "index_id_count";
    private static final String FIELD_HOST_ID = "host_id";
    private static final String FIELD_URL = "url";
    private static final String FIELD_HOST_NAME = "host_name";
    private static final String FIELD_URL_NAME = "url_name";
    private static final String FIELD_SUBMITTED_ON = "submitted_on";
    private static final String FIELD_PROCESSED_ON = "processed_on";
    private static final String FIELD_REAL_PROCESSED_ON = "real_processed_on";
    private static final String FIELD_FROM_ROBOTS_TXT = "from_robots_txt";
    private static final String FIELD_FROM_USER = "from_user";
    private static final String FIELD_FORMAT = "format";
    private static final String FIELD_URL_COUNT = "url_count";
    private static final String FIELD_XML_ERROR_COUNT = "xml_error_count";
    private static final String FIELD_URL_ERROR_COUNT = "url_error_count";
    private static final String FIELD_WARNING_COUNT = "warning_count";
    private static final String FIELD_CHILDREN_COUNT = "children_count";
    public static final String FIELD_TYPE = "type";
    public static final String FIELD_SEVERITY = "severity";
    public static final String FIELD_LINE = "line";
    public static final String FIELD_POSITION = "position";
    public static final String FIELD_TEXT = "text";

    private static final String SELECT_USER_SITEMAPS_QUERY =
            "SELECT " +
                    "    s.sitemap_id AS " + FIELD_SITEMAP_ID + ", " +
                    "    (SELECT si.index_id FROM tbl_sitemaps_indexes si WHERE si.sitemap_id = s.sitemap_id LIMIT 1) AS " + FIELD_INDEX_ID + ", " +
                    "    (SELECT COUNT(si.index_id) FROM tbl_sitemaps_indexes si WHERE si.sitemap_id = s.sitemap_id) AS " + FIELD_INDEX_ID_COUNT + ", " +
                    "    s.host_name AS " + FIELD_HOST_NAME + ", " +
                    "    s.url_name AS " + FIELD_URL_NAME + ", " +
                    "    s.from_robots_txt AS " + FIELD_FROM_ROBOTS_TXT + ", " +
                    "    s.from_user AS " + FIELD_FROM_USER + ", " +
                    "    s.submitted_on AS " + FIELD_SUBMITTED_ON + " " +
                    "FROM " +
                    "    tbl_sitemaps s " +
                    "WHERE " +
                    "    s.host_id = ? " +
                    "AND " +
                    "    s.from_user = 1 ";

    private static final String SELECT_SITEMAP_INFO_QUERY =
            "SELECT " +
                    "    s.sitemap_id AS " + FIELD_SITEMAP_ID + ", " +
                    "    (SELECT si.index_id FROM tbl_sitemaps_indexes si WHERE si.sitemap_id = s.sitemap_id LIMIT 1) AS " + FIELD_INDEX_ID + ", " +
                    "    (SELECT COUNT(si.index_id) FROM tbl_sitemaps_indexes si WHERE si.sitemap_id = s.sitemap_id) AS " + FIELD_INDEX_ID_COUNT + ", " +
                    "    s.host_name AS host_name, " +
                    "    s.url_name AS url_name, " +
                    "    s.submitted_on AS " + FIELD_SUBMITTED_ON + ", " +
                    "    s.processed_on AS " + FIELD_PROCESSED_ON + ", " +
                    "    s.real_processed_on AS " + FIELD_REAL_PROCESSED_ON + ", " +
                    "    s.from_robots_txt AS " + FIELD_FROM_ROBOTS_TXT + ", " +
                    "    s.from_user AS " + FIELD_FROM_USER + ", " +
                    "    s.format AS " + FIELD_FORMAT + ", " +
                    "    s.type AS " + FIELD_TYPE + ", " +
                    "    s.url_count AS " + FIELD_URL_COUNT + ", " +
                    "    (SELECT SUM(ces.count) FROM tbl_code_error_sitemap ces " +
                    "     WHERE ces.sitemap_id=s.sitemap_id group by ces.sitemap_id) AS " + FIELD_URL_ERROR_COUNT + ", " +
                    "    (SELECT COUNT(*) FROM tbl_sitemap_errors se JOIN tbl_dic_sitemap_error_type type ON se.type=type.id " +
                    "WHERE se.sitemap_id=s.sitemap_id AND type.severity=" + SeverityEnum.ERROR.getValue() + ") AS " + FIELD_XML_ERROR_COUNT + ", " +
                    "    (SELECT COUNT(*) FROM tbl_sitemap_errors se JOIN tbl_dic_sitemap_error_type type ON se.type=type.id " +
                    "WHERE se.sitemap_id=s.sitemap_id AND type.severity=" + SeverityEnum.WARNING.getValue() + ") AS " + FIELD_WARNING_COUNT + ", " +
                    "    (SELECT COUNT(*) FROM tbl_sitemaps_indexes si WHERE si.host_id=s.host_id AND si.index_id=s.sitemap_id) AS " + FIELD_CHILDREN_COUNT + " " +
                    "FROM " +
                    "    tbl_sitemaps s " +
                    "WHERE " +
                    "    s.host_id = ? " +
                    "AND " +
                    "    s.sitemap_id = ?";

    private static final String INSERT_SITEMAP_QUERY =
            "INSERT INTO " +
                    "   tbl_sitemaps (host_id, url_name, host_name, submitted_on, from_robots_txt, from_user) " +
                    "VALUES (?, ?, ?, ?, false, true) ";

    private static final String UPDATE_FROM_USER_FLAG_QUERY =
            "UPDATE " +
                    "    tbl_sitemaps " +
                    "SET " +
                    "    from_user = ? " +
                    "WHERE " +
                    "    sitemap_id = ?";

    private static final String SELECT_SITEMAP_BY_HOST_ID_AND_URL_QUERY =
            "SELECT " +
                    "    sitemaps.sitemap_id AS " + FIELD_SITEMAP_ID + ", " +
                    "    sitemaps.host_name AS host_name, " +
                    "    sitemaps.url_name AS url_name," +
                    "    sitemaps.submitted_on AS " + FIELD_SUBMITTED_ON + ", " +
                    "    sitemaps.processed_on AS " + FIELD_PROCESSED_ON + ", " +
                    "    sitemaps.real_processed_on AS " + FIELD_REAL_PROCESSED_ON + ", " +
                    "    sitemaps.from_robots_txt AS " + FIELD_FROM_ROBOTS_TXT + ", " +
                    "    sitemaps.from_user AS " + FIELD_FROM_USER + ", " +
                    "    types.name AS " + FIELD_FORMAT + ", " +
                    "    sitemaps.url_count AS " + FIELD_URL_COUNT + ", " +
                    "    (SELECT SUM(ces.count) FROM tbl_code_error_sitemap ces " +
                    "     WHERE ces.sitemap_id=sitemaps.sitemap_id group by ces.sitemap_id) AS " + FIELD_URL_ERROR_COUNT + " " +
                    "FROM " +
                    "    tbl_sitemaps sitemaps " +
                    "LEFT JOIN " +
                    "    tbl_dic_sitemap_type types " +
                    "ON " +
                    "    sitemaps.type=types.id " +
                    "WHERE " +
                    "    url_name = BINARY ? " +
                    "AND host_name = ? " +
                    "AND host_id = ?";

    private static final String SELECT_FULL_SITEMAP_INFO_BY_HOST_ID_AND_URL_QUERY =
            "SELECT " +
                    "    s.sitemap_id AS " + FIELD_SITEMAP_ID + ", " +
                    "    (SELECT si.index_id FROM tbl_sitemaps_indexes si WHERE si.sitemap_id = s.sitemap_id LIMIT 1) AS " + FIELD_INDEX_ID + ", " +
                    "    (SELECT COUNT(si.index_id) FROM tbl_sitemaps_indexes si WHERE si.sitemap_id = s.sitemap_id) AS " + FIELD_INDEX_ID_COUNT + ", " +
                    "    s.host_name AS host_name, " +
                    "    s.url_name AS url_name, " +
                    "    s.submitted_on AS " + FIELD_SUBMITTED_ON + ", " +
                    "    s.processed_on AS " + FIELD_PROCESSED_ON + ", " +
                    "    s.real_processed_on AS " + FIELD_REAL_PROCESSED_ON + ", " +
                    "    s.from_robots_txt AS " + FIELD_FROM_ROBOTS_TXT + ", " +
                    "    s.from_user AS " + FIELD_FROM_USER + ", " +
                    "    s.format AS " + FIELD_FORMAT + ", " +
                    "    s.type AS " + FIELD_TYPE + ", " +
                    "    s.url_count AS " + FIELD_URL_COUNT + ", " +
                    "    (SELECT SUM(ces.count) FROM tbl_code_error_sitemap ces " +
                    "     WHERE ces.sitemap_id=s.sitemap_id group by ces.sitemap_id) AS " + FIELD_URL_ERROR_COUNT + ", " +
                    "    (SELECT COUNT(*) FROM tbl_sitemap_errors se JOIN tbl_dic_sitemap_error_type type ON se.type=type.id " +
                    "WHERE se.sitemap_id=s.sitemap_id AND type.severity=" + SeverityEnum.ERROR.getValue() + ") AS " + FIELD_XML_ERROR_COUNT + ", " +
                    "    (SELECT COUNT(*) FROM tbl_sitemap_errors se JOIN tbl_dic_sitemap_error_type type ON se.type=type.id " +
                    "WHERE se.sitemap_id=s.sitemap_id AND type.severity=" + SeverityEnum.WARNING.getValue() + ") AS " + FIELD_WARNING_COUNT + ", " +
                    "    (SELECT COUNT(*) FROM tbl_sitemaps_indexes si WHERE si.host_id=s.host_id AND si.index_id=s.sitemap_id) AS " + FIELD_CHILDREN_COUNT + " " +
                    "FROM " +
                    "    tbl_sitemaps s " +
                    "LEFT JOIN " +
                    "    tbl_dic_sitemap_type types " +
                    "ON " +
                    "    s.type=types.id " +
                    "WHERE " +
                    "    url_name = BINARY ? " +
                    "AND host_name = ? " + // host_name в sitemap может отличаться от имени хоста
                    "AND host_id = ?";

    private static final String SELECT_SITEMAPS_HOST_IDS_QUERY =
            "SELECT " +
                    "    DISTINCT host_id AS " + FIELD_HOST_ID + " " +
                    "FROM " +
                    "    tbl_sitemaps " +
                    "WHERE " +
                    "    sitemap_id = ?";

    private static final String SELECT_SHORT_SITEMAP_INFO_QUERY =
            "SELECT " +
                    "    sitemap_id AS " + FIELD_SITEMAP_ID + ", " +
                    "    (SELECT si.index_id FROM tbl_sitemaps_indexes si WHERE si.sitemap_id = s.sitemap_id LIMIT 1) AS " + FIELD_INDEX_ID + ", " +
                    "    (SELECT COUNT(si.index_id) FROM tbl_sitemaps_indexes si WHERE si.sitemap_id = s.sitemap_id) AS " + FIELD_INDEX_ID_COUNT + ", " +
                    "    from_robots_txt AS " + FIELD_FROM_ROBOTS_TXT + ", " +
                    "    from_user AS " + FIELD_FROM_USER + " " +
                    "FROM " +
                    "    tbl_sitemaps s " +
                    "WHERE " +
                    "    sitemap_id = ?";

    private static final String DELETE_SITEMAP_QUERY =
            "DELETE FROM " +
                    "    tbl_sitemaps " +
                    "WHERE " +
                    "    sitemap_id = ? ";

    private static final String DELETE_SITEMAPS_INDICES_QUERY =
            "DELETE FROM " +
                    "    tbl_sitemaps_indexes " +
                    "WHERE " +
                    "    index_id = ? " +
                    "AND " +
                    "    host_id = ? " +
                    "AND " +
                    "    sitemap_id in " +
                    "(SELECT w.sitemap_id FROM " +
                    "(SELECT s.sitemap_id " +
                    "    FROM " +
                    "        tbl_sitemaps s " +
                    "    JOIN " +
                    "        tbl_sitemaps_indexes si " +
                    "    ON " +
                    "        s.sitemap_id = si.sitemap_id " +
                    "    WHERE " +
                    "        si.index_id = ? " +
                    "    AND " +
                    "        si.host_id = ? " +
                    ") w " +
                    ")";

    private static final String SELECT_INDEXES_QUERY =
            "SELECT " +
                    "   si.index_id AS " + FIELD_INDEX_ID + ", " +
                    "   s.url_name AS " + FIELD_URL_NAME + ", " +
                    "   s.host_name AS " + FIELD_HOST_NAME + " " +
                    "FROM " +
                    "   tbl_sitemaps_indexes si " +
                    "JOIN " +
                    "   tbl_sitemaps s " +
                    "ON " +
                    "   si.index_id = s.sitemap_id " +
                    "WHERE " +
                    "   si.sitemap_id = ? " +
                    "ORDER BY s.host_name ASC, s.url_name ASC";

    private HostDbHostInfoService hostDbHostInfoService;
    private HistoryService historyService;
    private TblRemovedUserSitemapsDao tblRemovedUserSitemapsDao;

    private SitemapInfo getSitemapInfo(HostDbHostInfo hostDbHostInfo, URL url) throws InternalException {
        List<SitemapInfo> list = getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).query(
                SELECT_SITEMAP_BY_HOST_ID_AND_URL_QUERY, briefSitemapInfoRowMapper, URLUtil.getRelativeUrl(url), getHostName(url), hostDbHostInfo.getHostDbHostId());
        return list.isEmpty() ? null : list.get(0);
    }

    public MergedSitemapInfo getSitemapInfoByUrl(HostDbHostInfo hostDbHostInfo, MergedSitemapInfo latestSitemapInfo) throws InternalException, UserException {
//        URL u;
//        try {
//            u = new URL(url);
//        } catch (MalformedURLException e) {
//            throw new InternalException(InternalProblem.INTERNAL_PROBLEM, "Can't create url from string " + url);
//        }
        List<MergedSitemapInfo> list = getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).query(
                SELECT_FULL_SITEMAP_INFO_BY_HOST_ID_AND_URL_QUERY, sitemapInfoRowMapper,
                latestSitemapInfo.getSitemapPath(), latestSitemapInfo.getSitemapHost(), hostDbHostInfo.getHostDbHostId());
        for (MergedSitemapInfo info : list) {
            if (info.getIndexIdCount() != null && info.getIndexIdCount() > 0) {
                info.setIndexId(calculateIndexIdForSitemapId(hostDbHostInfo, info.getId(), null));
            }
        }
        return list.isEmpty() ? null : list.get(0);
    }

    private SitemapInfo getSitemapInfo(HostDbHostInfo hostDbHostInfo, Long sitemapId) throws InternalException {
        String q = "SELECT " +
                "    sitemaps.host_id AS host_id, " +
                "    sitemaps.sitemap_id AS " + FIELD_SITEMAP_ID + ", " +
                "    sitemaps.host_name AS host_name, " +
                "    sitemaps.url_name AS url_name," +
                "    sitemaps.submitted_on AS " + FIELD_SUBMITTED_ON + ", " +
                "    sitemaps.processed_on AS " + FIELD_PROCESSED_ON + ", " +
                "    sitemaps.real_processed_on AS " + FIELD_REAL_PROCESSED_ON + ", " +
                "    sitemaps.from_robots_txt AS " + FIELD_FROM_ROBOTS_TXT + ", " +
                "    sitemaps.from_user AS " + FIELD_FROM_USER + ", " +
                "    types.name AS " + FIELD_FORMAT + ", " +
                "    sitemaps.url_count AS " + FIELD_URL_COUNT + ", " +
                "    0 AS " + FIELD_URL_ERROR_COUNT + " " +
                "FROM " +
                "    tbl_sitemaps sitemaps " +
                "LEFT JOIN " +
                "    tbl_dic_sitemap_type types " +
                "ON " +
                "    sitemaps.type=types.id " +
                "WHERE " +
                "    sitemaps.sitemap_id = ?";
        return getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).safeQueryForObject(q, briefSitemapInfoRowMapper, sitemapId);
    }

    private ShortSitemapInfo getShortSitemapInfo(HostDbHostInfo hostDbHostInfo, Long sitemapId) throws InternalException, UserException {
        List<ShortSitemapInfo> infos = getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).query(
                SELECT_SHORT_SITEMAP_INFO_QUERY, shortSitemapInfoRowMapper, sitemapId);

        for (ShortSitemapInfo info : infos) {
            if (info.getIndexIdCount() != null && info.getIndexIdCount() > 0) {
                info.setIndexId(calculateIndexIdForSitemapId(hostDbHostInfo, info.getId(), null));
            }
        }

        return infos.isEmpty() ? null : infos.get(0);
    }

    public SitemapInfo addSitemap(final long userId, final Long realUserId, final HostDbHostInfo hostDbHostInfo,
                                  final URL url, final Date submittedOn) throws UserException, InternalException {
        SitemapInfo info = (SitemapInfo) getServiceTransactionTemplate(new WMCPartition(hostDbHostInfo, null))
                .executeInService(new ServiceTransactionCallback() {
                    @Override
                    public SitemapInfo doInTransaction(TransactionStatus transactionStatus)
                            throws InternalException, UserException {
                        // try to get sitemap id from wmconsole DB
                        SitemapInfo info = getSitemapInfo(hostDbHostInfo, url);

                        if (info != null) {
                            if (info.isFromUser()) {
                                throw new UserException(WMCUserProblem.SITEMAP_ALREADY_EXISTS,
                                        "Sitemap already added!");
                            }
                            getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).update(
                                    UPDATE_FROM_USER_FLAG_QUERY, true, info.getId());
                        } else {
                            getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).update(
                                    INSERT_SITEMAP_QUERY, hostDbHostInfo.getHostDbHostId(),
                                    URLUtil.getRelativeUrl(url), getHostName(url), submittedOn);
                            info = getSitemapInfo(hostDbHostInfo, url);
                        }
                        return info;
                    }
                });

        try {
            historyService.addEvents(realUserId, userId, HistoryActionEnum.ADD, WMCHistoryObjectTypeEnum.SITEMAP,
                    info.getId());
        } catch (InternalException e) {
            log.error("Failed to save history (user " + userId + " added sitemap " + info.getId() + " " +
                    "for host " + hostDbHostInfo.getName() + ")", e);
        }

        return info;
    }

    /**
     * Получить имя со схемой
     *
     * @param url
     * @return
     */
    private String getHostName(URL url) {
        return URLUtil.getHostName(url, true);
    }

    public SitemapInfo addSitemap(final long userId, final Long realUserId, final BriefHostInfo hostInfo,
                                  final URL url) throws UserException, InternalException {
        return addSitemap(userId, realUserId, hostDbHostInfoService.getHostDbHostInfo(hostInfo.getName()), url, new Date());
    }

    public void internalRemoveSitemap(final HostDbHostInfo hostDbHostInfo, final long userId, final Long sitemapId)
            throws UserException, InternalException {
        getServiceTransactionTemplate(new WMCPartition(hostDbHostInfo, null)).executeInService(
                new ServiceTransactionCallback() {
                    @Override
                    public Object doInTransaction(TransactionStatus transactionStatus) throws InternalException, UserException {
                        getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).update(
                                UPDATE_FROM_USER_FLAG_QUERY, false, sitemapId);
                        ShortSitemapInfo info = getShortSitemapInfo(hostDbHostInfo, sitemapId);
                        if ((info != null) && !info.isFromRobotsTxt() && !info.isFromUser() && (info.getIndexId() == null)) {
                            getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).update(
                                    DELETE_SITEMAPS_INDICES_QUERY,
                                    sitemapId, hostDbHostInfo.getHostDbHostId(),
                                    sitemapId, hostDbHostInfo.getHostDbHostId());
                            getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).update(
                                    DELETE_SITEMAP_QUERY, sitemapId);
                        }
                        return null;
                    }
                });

        try {
            historyService.addEvents(userId, userId, HistoryActionEnum.REMOVE,
                    WMCHistoryObjectTypeEnum.SITEMAP, sitemapId);
        } catch (InternalException e) {
            log.error("Failed to save history (user " + userId + " removed sitemap " + sitemapId + " " +
                    "for host " + hostDbHostInfo.getName() + ")", e);
        }
    }

    public SitemapInfo removeSitemap(final HostDbHostInfo hostDbHostInfo, final BriefHostInfo hostInfo, final long userId, final Long realUserId, final Long sitemapId)
            throws UserException, InternalException {
        SitemapInfo info = getServiceTransactionTemplate(new WMCPartition(hostDbHostInfo, null)).executeInService(
                new ServiceTransactionCallback<SitemapInfo>() {
                    @Override
                    public SitemapInfo doInTransaction(TransactionStatus transactionStatus) throws InternalException, UserException {
                        getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).update(
                                UPDATE_FROM_USER_FLAG_QUERY, false, sitemapId);
                        SitemapInfo info = getSitemapInfo(hostDbHostInfo, sitemapId);
                        if ((info != null) && !info.isFromRobotsTxt() && !info.isFromUser() && (info.getIndexId() == null)) {
                            getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).update(
                                    DELETE_SITEMAPS_INDICES_QUERY,
                                    sitemapId, hostDbHostInfo.getHostDbHostId(),
                                    sitemapId, hostDbHostInfo.getHostDbHostId());
                            getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).update(
                                    DELETE_SITEMAP_QUERY, sitemapId);
                            // Эта таблица находится в другой базе, поэтому особо никак не связана с текущей транзакцией.
                            // Но если запись в эту таблицу обломится, то транзакция на первой базе откатится - поэтому все должно работать как надо
                            tblRemovedUserSitemapsDao.addRemovedSitemap(hostInfo, info.getUrl(), userId);
                        }
                        return info;
                    }
                });

        try {
            historyService.addEvents(realUserId, userId, HistoryActionEnum.REMOVE,
                    WMCHistoryObjectTypeEnum.SITEMAP, sitemapId);
        } catch (InternalException e) {
            log.error("Failed to save history (user " + userId + " removed sitemap " + sitemapId + " " +
                    "for host " + hostDbHostInfo.getName() + ")", e);
        }
        return info;
    }

    public void removeSitemaps(final HostDbHostInfo hostDbHostInfo, final long userId, final Long realUserId, final List<Long> sitemapIds)
            throws UserException, InternalException {

        getServiceTransactionTemplate(new WMCPartition(hostDbHostInfo, null)).executeInService(
                new ServiceTransactionCallback() {
                    @Override
                    public Object doInTransaction(TransactionStatus transactionStatus) throws InternalException, UserException {

                        for (Long sitemapId : sitemapIds) {
                            getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).update(
                                    UPDATE_FROM_USER_FLAG_QUERY, false, sitemapId);
                            ShortSitemapInfo info = getShortSitemapInfo(hostDbHostInfo, sitemapId);
                            if ((info != null) && !info.isFromRobotsTxt() && !info.isFromUser() && (info.getIndexId() == null)) {
                                getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).update(
                                        DELETE_SITEMAPS_INDICES_QUERY,
                                        sitemapId, hostDbHostInfo.getHostDbHostId(),
                                        sitemapId, hostDbHostInfo.getHostDbHostId());
                                getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).update(
                                        DELETE_SITEMAP_QUERY, sitemapId);
                            }
                        }

                        return null;
                    }
                });

        try {
            for (Long sitemapId : sitemapIds) {
                historyService.addEvents(realUserId, userId, HistoryActionEnum.REMOVE,
                        WMCHistoryObjectTypeEnum.SITEMAP, sitemapId);
            }
        } catch (InternalException e) {
            log.error("Failed to save history (user " + userId + " removed sitemap " + sitemapIds + " " +
                    "for host " + hostDbHostInfo.getName() + ")", e);
        }
    }

    public List<Long> getSitemapsHostIds(final HostDbHostInfo hostDbHostInfo, Long sitemapId) throws InternalException {
        return getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).query(SELECT_SITEMAPS_HOST_IDS_QUERY,
                new LongRowMapper(), sitemapId);
    }

    public List<ExtendedSitemapInfo> getUserAddedSitemapUrls(HostDbHostInfo hostInfo) throws InternalException, UserException {
        List<ExtendedSitemapInfo> res = getJdbcTemplate(new WMCPartition(hostInfo, null)).query(
                SELECT_USER_SITEMAPS_QUERY, extendedSitemapInfoRowMapper, hostInfo.getHostDbHostId());
        for (ExtendedSitemapInfo info : res) {
            if (info.getIndexIdCount() != null && info.getIndexIdCount() > 0) {
                info.setIndexId(calculateIndexIdForSitemapId(hostInfo, info.getId(), null));
            }
        }
        return res;
    }

    public MergedSitemapInfo getSitemapInfo2(
            final HostDbHostInfo hostDbHostInfo, final long sitemapId, final Long indexId)
            throws InternalException, UserException {
        final List<MergedSitemapInfo> res = getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).query(
                SELECT_SITEMAP_INFO_QUERY, sitemapInfoRowMapper, hostDbHostInfo.getHostDbHostId(), sitemapId);
        for (MergedSitemapInfo info : res) {
            if (info.getIndexIdCount() != null && info.getIndexIdCount() > 1) {
                info.setIndexId(calculateIndexIdForSitemapId(hostDbHostInfo, sitemapId, indexId));
            }
        }
        return res.isEmpty() ? null : res.get(0);
    }

    public Map<Long, List<SitemapInfo>> listUserSitemapsForHostsInOneDB(List<HostDbHostInfo> hosts) throws InternalException {
        if (hosts.isEmpty()) {
            return Collections.emptyMap();
        }
        Map<Integer, List<HostDbHostInfo>> partition2HostInfo = new HashMap<>();
        for (HostDbHostInfo hostDbHostInfo : hosts) {
            int partitionId = (int) (hostDbHostInfo.getHostDbHostId() % WMCPartition.PARTITIONS_COUNT);
            List<HostDbHostInfo> hostsForPartition = partition2HostInfo.get(partitionId);
            if (hostsForPartition == null) {
                hostsForPartition = new ArrayList<>();
                partition2HostInfo.put(partitionId, hostsForPartition);
            }
            hostsForPartition.add(hostDbHostInfo);
        }
        final Map<Long, List<SitemapInfo>> result = new HashMap<>();

        for (List<HostDbHostInfo> hostsForPartition : partition2HostInfo.values()) {
            Long[] params = new Long[hostsForPartition.size()];
            for (int i = 0; i < hostsForPartition.size(); i++) {
                params[i] = hostsForPartition.get(i).getHostDbHostId();
            }
            String q = "SELECT " +
                    "    sitemaps.host_id AS host_id, " +
                    "    sitemaps.sitemap_id AS " + FIELD_SITEMAP_ID + ", " +
                    "    sitemaps.host_name AS host_name, " +
                    "    sitemaps.url_name AS url_name," +
                    "    sitemaps.submitted_on AS " + FIELD_SUBMITTED_ON + ", " +
                    "    sitemaps.processed_on AS " + FIELD_PROCESSED_ON + ", " +
                    "    sitemaps.real_processed_on AS " + FIELD_REAL_PROCESSED_ON + ", " +
                    "    sitemaps.from_robots_txt AS " + FIELD_FROM_ROBOTS_TXT + ", " +
                    "    sitemaps.from_user AS " + FIELD_FROM_USER + ", " +
                    "    types.name AS " + FIELD_FORMAT + ", " +
                    "    sitemaps.url_count AS " + FIELD_URL_COUNT + ", " +
                    "    0 AS " + FIELD_URL_ERROR_COUNT + " " +
                    "FROM " +
                    "    tbl_sitemaps sitemaps " +
                    "LEFT JOIN " +
                    "    tbl_dic_sitemap_type types " +
                    "ON " +
                    "    sitemaps.type=types.id " +
                    "WHERE " +
                    "    sitemaps.from_user AND" +
                    "    sitemaps.host_id IN (" + SqlUtil.createQuestionMarks(hostsForPartition.size()) + ")";
            getJdbcTemplate(new WMCPartition(hostsForPartition.get(0), null))
                    .query(q, new RowCallbackHandler() {
                        @Override
                        public void processRow(ResultSet rs) throws SQLException {
                            SitemapInfo sitemapInfo = briefSitemapInfoRowMapper.mapRow(rs, 0);
                            long hostId = rs.getLong("host_id");
                            List<SitemapInfo> sitemapsForHost = result.get(hostId);
                            if (sitemapsForHost == null) {
                                sitemapsForHost = new ArrayList<>();
                                result.put(hostId, sitemapsForHost);
                            }
                            sitemapsForHost.add(sitemapInfo);
                        }
                    }, params);
        }
        return result;
    }

    private Long calculateIndexIdForSitemapId(HostDbHostInfo hostDbHostInfo, long sitemapId, Long indexId) throws InternalException, UserException {
        List<SitemapIndexInfo> indexList = getJdbcTemplate(new WMCPartition(hostDbHostInfo, null)).query(
                SELECT_INDEXES_QUERY, sitemapIndexMapper, sitemapId);

        if (!indexList.isEmpty()) {
            // вначале попытаемся найти indexId, переданный через параметры (если есть)
            if (indexId != null) {
                for (SitemapIndexInfo indexInfo : indexList) {
                    if (indexId.equals(indexInfo.getIndexId())) {
                        return indexId;
                    }
                }
            }

            // затем попытаемся найти sitemapIndex для того же хоста
            for (SitemapIndexInfo info : indexList) {
                String hostName = URLUtil.getHostName(
                        AbstractServantlet.prepareUrl(info.getHostName(), true), false);
                if (hostName.equals(hostDbHostInfo.getName())) {
                    return info.getIndexId();
                }
            }
            return indexList.iterator().next().getIndexId();
        }

        return null;
    }

    private static final ParameterizedRowMapper<SitemapIndexInfo> sitemapIndexMapper = new ParameterizedRowMapper<SitemapIndexInfo>() {
        @Override
        public SitemapIndexInfo mapRow(ResultSet rs, int rowNum) throws SQLException {
            return new SitemapIndexInfo(
                    rs.getLong(FIELD_INDEX_ID),
                    rs.getString(FIELD_URL_NAME),
                    rs.getString(FIELD_HOST_NAME));
        }
    };
    private static final ParameterizedRowMapper<SitemapInfo> briefSitemapInfoRowMapper = new ParameterizedRowMapper<SitemapInfo>() {
        @Override
        public SitemapInfo mapRow(ResultSet rs, int i) throws SQLException {
            return new SitemapInfo(
                    rs.getLong(FIELD_SITEMAP_ID),
                    rs.getString("host_name"),
                    rs.getString("url_name"),
                    SqlUtil.safeGetTimestamp(rs, FIELD_SUBMITTED_ON),
                    rs.getBoolean(FIELD_FROM_ROBOTS_TXT),
                    rs.getBoolean(FIELD_FROM_USER)
            );
        }
    };

    private static final ParameterizedRowMapper<MergedSitemapInfo> sitemapInfoRowMapper =
            new ParameterizedRowMapper<MergedSitemapInfo>() {
                @Override
                public MergedSitemapInfo mapRow(ResultSet rs, int i) throws SQLException {
                    Date submittedOn = SqlUtil.safeGetTimestamp(rs, FIELD_SUBMITTED_ON);
                    if (rs.wasNull() || submittedOn != null && submittedOn.getTime() <= 0) {
                        submittedOn = null;
                    }

                    Date processedOn = SqlUtil.safeGetTimestamp(rs, FIELD_PROCESSED_ON);
                    if (rs.wasNull()) {
                        processedOn = null;
                    }

                    Date realProcessedOn = SqlUtil.safeGetTimestamp(rs, FIELD_REAL_PROCESSED_ON);
                    if (rs.wasNull()) {
                        realProcessedOn = null;
                    }

                    SitemapFormatEnum format = SitemapFormatEnum.getByValue(SqlUtil.getIntNullable(rs, FIELD_FORMAT));

                    SitemapTypeEnum type = SitemapTypeEnum.getByValue(SqlUtil.getIntNullable(rs, FIELD_TYPE));

                    Integer urlsCount = SqlUtil.getIntNullable(rs, FIELD_URL_COUNT);

                    Long indexId = SqlUtil.getLongNullable(rs, FIELD_INDEX_ID);
                    Long indexIdCount = SqlUtil.getLongNullable(rs, FIELD_INDEX_ID_COUNT);

                    Integer urlErrorsCount = (processedOn == null) ? null : rs.getInt(FIELD_URL_ERROR_COUNT);
                    Integer xmlErrorsCount = (processedOn == null) ? null : rs.getInt(FIELD_XML_ERROR_COUNT);
                    Integer warningsCount = (processedOn == null) ? null : rs.getInt(FIELD_WARNING_COUNT);
                    int childrenCount = rs.getInt(FIELD_CHILDREN_COUNT);

                    String hostName = rs.getString("host_name");
                    String sitemapPath = rs.getString("url_name");

                    return new MergedSitemapInfo(
                            rs.getLong(FIELD_SITEMAP_ID),
                            indexId,
                            indexIdCount,
                            hostName,
                            sitemapPath,
                            submittedOn,
                            rs.getBoolean(FIELD_FROM_ROBOTS_TXT),
                            rs.getBoolean(FIELD_FROM_USER),
                            processedOn,
                            realProcessedOn,
                            format,
                            type,
                            urlsCount,
                            xmlErrorsCount,
                            urlErrorsCount,
                            warningsCount,
                            childrenCount > 0,
                            SitemapSourceEnum.IN_SEARCH
                    );
                }
            };

    private static final ParameterizedRowMapper<ShortSitemapInfo> shortSitemapInfoRowMapper =
            new ParameterizedRowMapper<ShortSitemapInfo>() {
                @Override
                public ShortSitemapInfo mapRow(ResultSet rs, int i) throws SQLException {
                    Long indexId = SqlUtil.getLongNullable(rs, FIELD_INDEX_ID);
                    Long indexIdCount = SqlUtil.getLongNullable(rs, FIELD_INDEX_ID_COUNT);

                    return new ShortSitemapInfo(
                            rs.getLong(FIELD_SITEMAP_ID),
                            indexId,
                            indexIdCount,
                            rs.getBoolean(FIELD_FROM_ROBOTS_TXT),
                            rs.getBoolean(FIELD_FROM_USER)
                    );
                }
            };

    private static final ParameterizedRowMapper<ExtendedSitemapInfo> extendedSitemapInfoRowMapper =
            new ParameterizedRowMapper<ExtendedSitemapInfo>() {
                @Override
                public ExtendedSitemapInfo mapRow(ResultSet rs, int i) throws SQLException {
                    Long indexId = SqlUtil.getLongNullable(rs, FIELD_INDEX_ID);
                    Long indexIdCount = SqlUtil.getLongNullable(rs, FIELD_INDEX_ID_COUNT);

                    final String sitemapHostName = rs.getString(FIELD_HOST_NAME);
                    final String sitemapUrlName = rs.getString(FIELD_URL_NAME);
                    final Date submittedOn = SqlUtil.safeGetTimestamp(rs, FIELD_SUBMITTED_ON);

                    return new ExtendedSitemapInfo(
                            rs.getLong(FIELD_SITEMAP_ID),
                            indexId,
                            indexIdCount,
                            rs.getBoolean(FIELD_FROM_ROBOTS_TXT),
                            rs.getBoolean(FIELD_FROM_USER),
                            sitemapHostName,
                            sitemapUrlName,
                            submittedOn
                    );
                }
            };

    @Required
    public void setHostDbHostInfoService(HostDbHostInfoService hostDbHostInfoService) {
        this.hostDbHostInfoService = hostDbHostInfoService;
    }

    @Required
    public void setHistoryService(HistoryService historyService) {
        this.historyService = historyService;
    }

    @Required
    public void setTblRemovedUserSitemapsDao(TblRemovedUserSitemapsDao tblRemovedUserSitemapsDao) {
        this.tblRemovedUserSitemapsDao = tblRemovedUserSitemapsDao;
    }
}
