import os.path
import sys
import json
import glob
import shutil
import urllib.request
import bbox_utils
import progress

TRAFFIC_SIGNS_NAME_MAP = {
    'sign_ru_1.1'                       : '',
    'sign_ru_1.2'                       : '',
    'sign_ru_1.4.1'                     : '',
    'sign_ru_1.4.4'                     : '',
    'sign_ru_1.4.5'                     : '',
    'sign_ru_1.5'                       : '',
    'sign_ru_1.6'                       : '',
    'sign_ru_1.7'                       : 'warning_roundabout_ahead',
    'sign_ru_1.8'                       : '',
    'sign_ru_1.10'                      : '',
    'sign_ru_1.11.1'                    : '',
    'sign_ru_1.11.2'                    : '',
    'sign_ru_1.12.1'                    : '',
    'sign_ru_1.12.2'                    : '',
    'sign_ru_1.14'                      : '',
    'sign_ru_1.15'                      : '',
    'sign_ru_1.16'                      : '',
    'sign_ru_1.17'                      : 'warning_uneven_road_ahead',
    'sign_ru_1.18'                      : '',
    'sign_ru_1.20.1'                    : 'warning_road_narrows_on_both',
    'sign_ru_1.20.2'                    : 'warning_road_narrows_on_right',
    'sign_ru_1.20.3'                    : 'warning_road_narrows_on_left',
    'sign_ru_1.21'                      : '',
    'sign_ru_1.22'                      : 'warning_pedestrian_crossing_ahead',
    'sign_ru_1.23'                      : 'warning_children',
    'sign_ru_1.24'                      : '',
    'sign_ru_1.25'                      : '',
    'sign_ru_1.31'                      : '',
    'sign_ru_1.33'                      : '',
    'sign_ru_1.34.1'                    : '',
    'sign_ru_1.34.2'                    : '',
    'sign_ru_1.34.3'                    : '',

    'sign_ru_2.1'                       : 'priority_priority_road',
    'sign_ru_2.2'                       : 'priority_eof_priority_road',
    'sign_ru_2.3.1'                     : 'warning_crossroads_minor_road_r_l',
    'sign_ru_2.3.2'                     : 'warning_crossroads_minor_road_r',
    'sign_ru_2.3.3'                     : 'warning_crossroads_minor_road_l',
    'sign_ru_2.3.4'                     : 'warning_crossroads_minor_road_fr',
    'sign_ru_2.3.5'                     : 'warning_crossroads_minor_road_fl',
    'sign_ru_2.3.6'                     : 'warning_crossroads_minor_road_br',
    'sign_ru_2.3.7'                     : 'warning_crossroads_minor_road_bl',
    'sign_ru_2.4'                       : 'priority_give_way',
    'sign_ru_2.5'                       : 'priority_stop',
    'sign_ru_2.6'                       : '',
    'sign_ru_2.7'                       : '',

    'sign_ru_3.1'                       : 'prohibitory_no_entry',
    'sign_ru_3.2'                       : 'prohibitory_no_vehicles',
    'sign_ru_3.3'                       : '',
    'sign_ru_3.4'                       : 'prohibitory_no_heavy_goods_vehicles',
    'sign_ru_3.7'                       : '',
    'sign_ru_3.9'                       : '',
    'sign_ru_3.10'                      : '',
    'sign_ru_3.11'                      : 'prohibitory_max_weight',
    'sign_ru_3.12'                      : 'prohibitory_max_weight_per_axle',
    'sign_ru_3.13'                      : 'prohibitory_max_height',
    'sign_ru_3.14'                      : 'prohibitory_max_width',
    'sign_ru_3.16'                      : '',
    'sign_ru_3.17.3'                    : '',
    'sign_ru_3.18.1'                    : 'prohibitory_no_right_turn',
    'sign_ru_3.18.2'                    : 'prohibitory_no_left_turn',
    'sign_ru_3.19'                      : 'prohibitory_no_uturn',
    'sign_ru_3.20'                      : 'prohibitory_no_overtaking',
    'sign_ru_3.21'                      : 'prohibitory_eof_no_overtaking',
    'sign_ru_3.22'                      : 'prohibitory_no_overtaking_by_heavy_vehicle',
    'sign_ru_3.23'                      : 'prohibitory_eof_no_overtaking_by_heavy_vehicle',
    'sign_ru_3.24_n5'                   : 'prohibitory_max_speed_5',
    'sign_ru_3.24_n10'                  : 'prohibitory_max_speed_10',
    'sign_ru_3.24_n15'                  : 'prohibitory_max_speed_15',
    'sign_ru_3.24_n20'                  : 'prohibitory_max_speed_20',
    'sign_ru_3.24_n25'                  : 'prohibitory_max_speed_25',
    'sign_ru_3.24_n30'                  : 'prohibitory_max_speed_30',
    'sign_ru_3.24_n35'                  : 'prohibitory_max_speed_35',
    'sign_ru_3.24_n40'                  : 'prohibitory_max_speed_40',
    'sign_ru_3.24_n45'                  : 'prohibitory_max_speed_45',
    'sign_ru_3.24_n50'                  : 'prohibitory_max_speed_50',
    'sign_ru_3.24_n55'                  : 'prohibitory_max_speed_55',
    'sign_ru_3.24_n60'                  : 'prohibitory_max_speed_60',
    'sign_ru_3.24_n70'                  : 'prohibitory_max_speed_70',
    'sign_ru_3.24_n80'                  : 'prohibitory_max_speed_80',
    'sign_ru_3.24_n90'                  : 'prohibitory_max_speed_90',
    'sign_ru_3.24_n100'                 : 'prohibitory_max_speed_100',
    'sign_ru_3.24_n110'                 : 'prohibitory_max_speed_110',
    'sign_ru_3.24_n120'                 : 'prohibitory_max_speed_120',
    'sign_ru_3.24_n130'                 : 'prohibitory_max_speed_130',
    'sign_ru_3.25'                      : 'prohibitory_eof_max_speed',
    'sign_ru_3.25_n5'                   : 'prohibitory_eof_max_speed_5',
    'sign_ru_3.25_n10'                  : 'prohibitory_eof_max_speed_10',
    'sign_ru_3.25_n15'                  : 'prohibitory_eof_max_speed_15',
    'sign_ru_3.25_n20'                  : 'prohibitory_eof_max_speed_20',
    'sign_ru_3.25_n25'                  : 'prohibitory_eof_max_speed_25',
    'sign_ru_3.25_n30'                  : 'prohibitory_eof_max_speed_30',
    'sign_ru_3.25_n35'                  : 'prohibitory_eof_max_speed_35',
    'sign_ru_3.25_n40'                  : 'prohibitory_eof_max_speed_40',
    'sign_ru_3.25_n45'                  : 'prohibitory_eof_max_speed_45',
    'sign_ru_3.25_n50'                  : 'prohibitory_eof_max_speed_50',
    'sign_ru_3.25_n55'                  : 'prohibitory_eof_max_speed_55',
    'sign_ru_3.25_n60'                  : 'prohibitory_eof_max_speed_60',
    'sign_ru_3.25_n70'                  : 'prohibitory_eof_max_speed_70',
    'sign_ru_3.25_n80'                  : 'prohibitory_eof_max_speed_80',
    'sign_ru_3.25_n90'                  : 'prohibitory_eof_max_speed_90',
    'sign_ru_3.25_n100'                 : 'prohibitory_eof_max_speed_100',
    'sign_ru_3.25_n110'                 : 'prohibitory_eof_max_speed_110',
    'sign_ru_3.25_n120'                 : 'prohibitory_eof_max_speed_120',
    'sign_ru_3.25_n130'                 : 'prohibitory_eof_max_speed_130',
    'sign_ru_3.26'                      : '',
    'sign_ru_3.27'                      : 'prohibitory_no_parking_or_stopping',
    'sign_ru_3.28'                      : 'prohibitory_no_parking',
    'sign_ru_3.29'                      : '',
    'sign_ru_3.30'                      : '',
    'sign_ru_3.31'                      : '',
    'sign_ru_3.32'                      : '',
    'sign_ru_4.1.1'                     : 'mandatory_proceed_straight',
    'sign_ru_4.1.2'                     : 'mandatory_turn_right_ahead',
    'sign_ru_4.1.3'                     : 'mandatory_turn_left_ahead',
    'sign_ru_4.1.4'                     : 'mandatory_proceed_straight_or_turn_right',
    'sign_ru_4.1.5'                     : 'mandatory_proceed_straight_or_turn_left',
    'sign_ru_4.1.6'                     : '',
    'sign_ru_4.2.1'                     : '',
    'sign_ru_4.2.2'                     : '',
    'sign_ru_4.2.3'                     : '',
    'sign_ru_4.3'                       : 'mandatory_roundabout',
    'sign_ru_4.4.1'                     : '',
    'sign_ru_4.4.2'                     : '',
    'sign_ru_4.5.1'                     : '',
    'sign_ru_4.5.2'                     : '',
    'sign_ru_4.5.4'                     : '',
    'sign_ru_4.5.5'                     : '',
    'sign_ru_4.6'                       : '',
    'sign_ru_4.8.1'                     : '',
    'sign_ru_4.8.3'                     : '',

    'sign_ru_5.3'                       : '',
    'sign_ru_5.4'                       : '',
    'sign_ru_5.5'                       : 'prescription_one_way_road',
    'sign_ru_5.6'                       : 'prescription_eof_one_way_road',
    'sign_ru_5.7.1'                     : 'prescription_entry_to_one_way_road_on_the_right',
    'sign_ru_5.7.2'                     : 'prescription_entry_to_one_way_road_on_the_left',

    'sign_ru_5.8'                       : '',
    'sign_ru_5.9'                       : '',
    'sign_ru_5.10'                      : '',
    'sign_ru_5.11.1'                    : 'prescription_bus_lane',
    'sign_ru_5.11.2'                    : '',
    'sign_ru_5.12.1'                    : 'prescription_eof_bus_lane',
    'sign_ru_5.12.2'                    : '',
    'sign_ru_5.13.1'                    : '',
    'sign_ru_5.13.2'                    : '',
    'sign_ru_5.13.3'                    : '',
    'sign_ru_5.14'                      : '',
    'sign_ru_5.14.1'                    : '',
    'sign_ru_5.14.2'                    : '',
    'sign_ru_5.14.3'                    : '',
    'sign_ru_5.15.1'                    : '',
    'sign_ru_5.15.2'                    : '',
    'sign_ru_5.15.2_forward'            : 'prescription_lane_direction_f',
    'sign_ru_5.15.2_forward_left'       : 'prescription_lane_direction_f_l',
    'sign_ru_5.15.2_forward_right'      : 'prescription_lane_direction_f_r',
    'sign_ru_5.15.2_forward_right_left' : '',
    'sign_ru_5.15.2_forward_round_left' : '',
    'sign_ru_5.15.2_left'               : 'prescription_lane_direction_l',
    'sign_ru_5.15.2_left_45'            : 'prescription_lane_direction_fl',
    'sign_ru_5.15.2_right'              : 'prescription_lane_direction_r',
    'sign_ru_5.15.2_right_45'           : 'prescription_lane_direction_fr',
    'sign_ru_5.15.2_right_left'         : 'prescription_lane_direction_r_l',
    'sign_ru_5.15.3'                    : 'prescription_start_new_line_right',
    'sign_ru_5.15.4'                    : 'prescription_start_new_line_left',
    'sign_ru_5.15.5'                    : 'prescription_end_line_right',
    'sign_ru_5.15.6'                    : 'prescription_end_line_left',
    'sign_ru_5.15.7'                    : '',
    'sign_ru_5.16'                      : '',
    'sign_ru_5.17'                      : '',
    'sign_ru_5.18'                      : '',
    'sign_ru_5.19.1'                    : '',
    'sign_ru_5.20'                      : '',
    'sign_ru_5.21'                      : '',
    'sign_ru_5.22'                      : '',
    'sign_ru_5.23.1'                    : 'prescription_built_up_area',
    'sign_ru_5.24.1'                    : 'prescription_eof_built_up_area',

    'sign_ru_6.2'                       : '',
    'sign_ru_6.3.1'                     : '',
    'sign_ru_6.3.2'                     : '',
    'sign_ru_6.4'                       : 'information_parking',
    'sign_ru_6.6'                       : '',
    'sign_ru_6.7'                       : '',
    'sign_ru_6.8.1'                     : '',
    'sign_ru_6.8.2'                     : '',
    'sign_ru_6.8.3'                     : '',
    'sign_ru_6.10.1'                    : '',
    'sign_ru_6.16'                      : '',

    'sign_ru_7.1'                       : '',
    'sign_ru_7.2'                       : '',
    'sign_ru_7.3'                       : '',
    'sign_ru_7.4'                       : '',
    'sign_ru_7.5'                       : '',
    'sign_ru_7.7'                       : '',
    'sign_ru_7.9'                       : '',
    'sign_ru_7.12'                      : '',
    'sign_ru_7.15'                      : '',

    'sign_ru_8.1.1'                     : '',
    'sign_ru_8.1.2'                     : 'information_distance_stop_ahead',
    'sign_ru_8.1.3'                     : 'information_distance_object_right',
    'sign_ru_8.1.4'                     : 'information_distance_object_left',
    'sign_ru_8.2.1'                     : '',
    'sign_ru_8.2.2'                     : 'information_start_zone',
    'sign_ru_8.2.3'                     : '',
    'sign_ru_8.2.4'                     : 'information_in_zone',
    'sign_ru_8.2.5'                     : 'information_distance_zone_right',
    'sign_ru_8.2.6'                     : 'information_distance_zone_left',
    'sign_ru_8.3.1'                     : '',
    'sign_ru_8.3.2'                     : '',
    'sign_ru_8.3.3'                     : '',
    'sign_ru_8.11'                      : '',
    'sign_ru_8.13'                      : '',
    'sign_ru_8.14'                      : '',
    'sign_ru_8.15'                      : '',
    'sign_ru_8.16'                      : '',
    'sign_ru_8.17'                      : '',
    'sign_ru_8.4.1'                     : 'information_heavy_vehicle',
    'sign_ru_8.4.3'                     : 'information_light_vehicle',
    'sign_ru_8.4.4'                     : '',
    'sign_ru_8.4.6'                     : '',
    'sign_ru_8.4.7'                     : '',
    'sign_ru_8.4.9'                     : '',
    'sign_ru_8.4.10'                    : '',
    'sign_ru_8.4.11'                    : '',
    'sign_ru_8.4.14'                    : '',
    'sign_ru_8.5.1'                     : 'information_holiday',
    'sign_ru_8.5.2'                     : 'information_working_day',
    'sign_ru_8.5.4'                     : '',
    'sign_ru_8.5.5'                     : 'information_holiday_time',
    'sign_ru_8.5.6'                     : 'information_working_day_time',
    'sign_ru_8.6.1'                     : '',
    'sign_ru_8.6.2'                     : '',
    'sign_ru_8.6.3'                     : '',
    'sign_ru_8.6.4'                     : '',
    'sign_ru_8.6.5'                     : '',
    'sign_ru_8.6.7'                     : '',
    'sign_ru_8.6.9'                     : '',
    'sign_ru_8.7'                       : '',
    'sign_ru_8.8'                       : 'information_paid_services',
    'sign_ru_8.22.1'                    : '',
    'sign_ru_8.22.2'                    : '',
    'sign_ru_8.22.3'                    : '',
    'sign_ru_8.23'                      : '',
    'sign_ru_8.24'                      : '',
};

def parse_image_url(url):
    # url must be in format:
    #    https://mrc-browser.maps.yandex.net/feature/<feature_id>/image
    # or
    #    https://mrc-browser.maps.yandex.net/feature/<feature_id>/undistorted
    URL_PREFIX = 'https://mrc-browser.maps.yandex.net/feature/'

    assert (URL_PREFIX == url[:len(URL_PREFIX)]), 'Wrong URL format {}'.format(url)
    feature_id = url[len(URL_PREFIX) : url.rfind('/')]
    suffix = url[url.rfind('/') + 1:]
    return feature_id, suffix

def json_union(json1, json2):
    csr1 = json1['classified_signs']
    csr2 = json2['classified_signs']

    result = dict()
    result['classified_signs'] = csr1 + csr2
    return result

def json_map_names(json_data, json_path_out = None):
    csr = json_data['classified_signs']
    for idx, result in enumerate(csr):
        signs    = result['signs']
        url      = result['source']
        for sign in signs:
            sign_id = sign['sign_id']
            if (sign_id in TRAFFIC_SIGNS_NAME_MAP) and (TRAFFIC_SIGNS_NAME_MAP[sign_id] != ''):
                sign['sign_id'] = TRAFFIC_SIGNS_NAME_MAP[sign_id]

    if (json_path_out is not None):
        with open(json_path_out, 'w') as f:
            json.dump(json_data, f)
    return json_data

def load_json(url, map_class = True):
    response = urllib.request.urlopen(url)
    if (response.headers.get_content_charset() is None):
        charset = 'utf-8'
    else:
        charset = response.headers.get_content_charset()
    return json_map_names(json.loads(response.read().decode(charset)))

def download_and_merge_json_simple(urls):
    result = None
    for url in urls:
        if (result is None):
            result = load_json(url)
        else:
            result = json_union(result, load_json(url))
    return result

def json_to_urls_dict(json_data):
    result = dict()
    csr = json_data['classified_signs']
    for item in enumerate(csr):
        url   = item['source']
        signs = item['signs']
        result[url] = signs
    return result

def merge_signs(signs1, signs2):
    MERGE_SIGNS_THR = 0.7

    result = []
    for sign2 in signs2:
        bbox2 = sign2['bbox']
        addsign = True
        for sign1 in signs1:
            bbox1 = sign1['bbox']
            if (bbox_utils.iou(bbox1, bbox2) < MERGE_SIGNS_THR):
                continue
            addsign = False
            if (sign1['sign_id'] == sign2['sign_id']):
                if (sign2['answer'] == 'ok'):
                    sign1['answer'] = 'ok'
                    sign1['bbox'] = sign2['bbox']#?????
                break
            if (sign1['answer'] != 'ok' and sign2['answer'] == 'ok'):
                sign1['sign_id'] = sign2['sign_id']
                sign1['answer'] = 'ok'
                sign1['bbox'] = sign2['bbox']#?????
                break
        if addsign:
            result += [sign2]
    result += signs1
    return result

def json_merge_to_urls_dict(urls_signs, json_data):
    csr = json_data['classified_signs']
    for item in csr:
        url   = item['source']
        signs = item['signs']
        if (url in urls_signs.items()):
            urls_signs[url] = merge_signs(urls_signs[url], signs)
        else:
            urls_signs[url] = signs
    return urls_signs

def download_and_merge_json_smart(urls):
    urls_signs = dict()
    for url in urls:
        if (urls_signs is None):
            urls_signs = json_to_urls_dict(load_json(url))
        else:
            urls_signs = json_merge_to_urls_dict(urls_signs, load_json(url))
    csr = []
    for item in urls_signs.items():
        csr += [{'source': item[0], 'signs' : item[1]}]
    result = {'classified_signs' : csr}
    return result

def download_and_merge_json(urls, rework_double_url = True, json_path_out = None):
    if (rework_double_url):
        result = download_and_merge_json_smart(urls)
    else:
        result = download_and_merge_json_simple(urls)

    if (json_path_out is not None):
        with open(json_path_out, 'w') as f:
            json.dump(result, f)
    return result

def download_and_merge_from_setting(settings_path, rework_double_url = True, json_path_out = None):
    with open(settings_path) as f:
        settings = json.load(f)
    urls = settings['urls']
    return download_and_merge_json(urls, rework_double_url, json_path_out)

def stat(json_data):
    csr = json_data['classified_signs']

    item_cnt = len(csr)
    answers = dict()
    sign_ids = dict()
    for idx, result in enumerate(csr):
        signs    = result['signs']
        url      = result['source']
        for sign in signs:
            answer  = sign['answer']
            if answer in answers:
                answers[answer] += 1
            else:
                answers[answer] = 1
            if (answer != 'ok'):
                continue
            sign_id = sign['sign_id']
            if sign_id in sign_ids:
                sign_ids[sign_id] += 1
            else:
                sign_ids[sign_id] = 1
    return answers, sign_ids

def print_stat(answers, sign_cnt, sign_list = None, cnt_thr = None):
    if (answers is not None):
        for answer in answers.items():
            print('| {:<25} | {:>5} |'.format(answer[0], answer[1]))
        print('---------------------------------')
    if (sign_list is None):
        sign_cnt_sorted = sorted(sign_cnt.items(), key=lambda kv: kv[0])
        for sign_id in sign_cnt_sorted:
            print('| {:<50} | {:>5} |'.format(sign_id[0], sign_id[1]))
    else:
        for sign_id in sign_list:
            cnt = 0
            if sign_id in sign_cnt:
                cnt = sign_cnt[sign_id]
            if (cnt_thr is None) or (cnt > cnt_thr):
                print('| {:<50} | {:>5} |'.format(sign_id, cnt))

def stat_file(json_path, sign_list = None, cnt_thr = None):
    with open(json_path) as f:
        json_data = json.load(f)
    return stat(json_data)

def extract_urls(json_path):
    with open(json_path) as f:
        json_data = json.load(f)
    csr = json_data['classified_signs']

    for idx, result in enumerate(csr):
        url = result['source']
        print(url)

def filter_classes(classes, sign_cnt, min_amount):
    classes_filtered = []
    for cl in classes:
        if (cl in sign_cnt and sign_cnt[cl] > min_amount):
            classes_filtered += [cl]
    return classes_filtered

def filter_json_by_classes(json_data, classes):
    csr = json_data['classified_signs']
    csr_filtered = []
    for idx, item in enumerate(csr):
        signs = item['signs']
        signs_valid = []
        for sign in signs:
            if (sign['answer'] != 'ok'):
                continue
            sign_id = sign['sign_id']
            if sign_id in classes:
                signs_valid += [sign]
        if (len(signs_valid) > 0):
            item_new = {'source': item['source'], 'signs' : signs_valid}
            csr_filtered += [item_new]
    result = {'classified_signs' : csr_filtered}
    return result

def filter_json_by_sizes(json_data, min_bbox_width, min_bbox_height):
    csr = json_data['classified_signs']
    csr_filtered = []
    for idx, item in enumerate(csr):
        signs = item['signs']
        signs_valid = []
        for sign in signs:
            if (sign['answer'] != 'ok'):
                continue
            bbox = sign['bbox']
            if (abs(bbox[1][0] - bbox[0][0]) > min_bbox_width) or (abs(bbox[1][1] - bbox[0][1]) > min_bbox_height):
                signs_valid += [sign]
        if (len(signs_valid) > 0):
            item_new = {'source': item['source'], 'signs' : signs_valid}
            csr_filtered += [item_new]
    result = {'classified_signs' : csr_filtered}
    return result

def print_urls(json_data):
    csr = json_data['classified_signs']
    for item in csr:
        print(item['source'])

def image_url_to_local_path(url, image_folder):
    # url must be in format:
    #    https://mrc-browser.maps.yandex.net/feature/<feature_id>/image
    # or
    #    https://mrc-browser.maps.yandex.net/feature/<feature_id>/undistorted
    URL_SUFFIXES = {'image' : '_i', 'undistorted' : '_u'}

    feature_id, suffix = parse_image_url(url)
    assert (suffix in URL_SUFFIXES), 'Wrong URL format {} {}'.format(url, suffix)

    image_path = '{}/{}{}.jpg'.format(image_folder, feature_id, URL_SUFFIXES[suffix])
    return feature_id, image_path

def download_images(json_data, image_folder):
    # url must be in format:
    #    https://mrc-browser.maps.yandex.net/feature/<feature_id>/image
    # or
    #    https://mrc-browser.maps.yandex.net/feature/<feature_id>/undistorted
    RETRY_COUNT  = 1

    if (not os.path.isdir(image_folder)):
        os.makedirs(image_folder)

    csr = json_data['classified_signs']
    cnt = len(csr)
    urls_downloaded = []
    urls_not_downloaded = []
    for idx, item in enumerate(csr):
        signs = item['signs']
        url   = item['source']

        if (url in urls_downloaded):
            raise Exception('Url {} already downloaded'.format(url))
        elif (url in urls_not_downloaded):
            raise Exception('Url {} had tried to download but error'.format(url))

        _, image_path = image_url_to_local_path(url, image_folder)
        if (os.path.isfile(image_path)):
            urls_downloaded += [url]
            continue
        downloaded = False
        for i in range(RETRY_COUNT):
            try:
                response = urllib.request.urlopen(url)
                data = response.read()
                with open(image_path,'wb') as img_out:
                    img_out.write(data)
                downloaded = True
                break
            except urllib.error.HTTPError:
                pass
        if (downloaded):
            urls_downloaded += [url]
        else:
            urls_not_downloaded += [url]
        progress.printProgressBar(idx+1, cnt, prefix = 'Download images: ', suffix = '')
    return urls_downloaded, urls_not_downloaded

def filter_json_by_urls(json_data, urls_valid):
    csr = json_data['classified_signs']
    csr_filtered = []
    for item in csr:
        if (item['source'] in urls_valid):
            csr_filtered += [item]
    result = {'classified_signs' : csr_filtered}
    return result

def search_double_urls(json_data):
    csr = json_data['classified_signs']
    urls = {}
    for idx, item in enumerate(csr):
        url, _ = parse_image_url(item['source'])
        if (url in urls):
            urls[url] += 1
        else:
            urls[url] = 1

    doubled_urls = []
    for url in urls.items():
        if (url[1] > 1):
            doubled_urls += [url[0]]
    return doubled_urls

def prepare_json(settings_path, image_folder, bad_urls_out_path = None, json_path_out = None):
    with open(settings_path) as f:
        settings = json.load(f)
    urls = settings['urls']

    main_json = download_and_merge_json(urls)
    _, sign_cnt = stat(main_json)

    classes = filter_classes(settings['classes'], sign_cnt, int(settings['min_amount']))
    out_json = filter_json_by_classes(main_json, classes)
    doubled_urls = search_double_urls(out_json)
    if (len(doubled_urls) > 0):
        print('Some URLS appears more one time:')
        for url in doubled_urls:
            print('  {}'.format(url))
        return None, None

    if (image_folder is not None):
        urls_downloaded, urls_not_downloaded = download_images(out_json, image_folder)
        if (bad_urls_out_path is not None):
            with open(bad_urls_out_path, 'w') as of:
                for url in urls_not_downloaded:
                    of.write('  {}\n'.format(url))
        out_json = filter_json_by_urls(out_json, urls_downloaded)
        _, sign_cnt = stat(out_json)
        classes = filter_classes(settings['classes'], sign_cnt, int(settings['min_amount']))
        out_json = filter_json_by_classes(out_json, classes)

    if (json_path_out is not None):
        with open(json_path_out, 'w') as f:
            json.dump(out_json, f)
    return out_json, classes

if __name__ == '__main__':
    if (len(sys.argv) <= 1):
       print("Operation not defined")
    elif ('download' == sys.argv[1]):
        download_and_merge_from_setting(sys.argv[2], json_path_out = sys.argv[3])
    elif ('stat' == sys.argv[1]):
        answers, sign_ids = stat_file(sys.argv[2])
        print_stat(answers, sign_ids)
    elif ('extracturls' == sys.argv[1]):
        extract_urls(sys.argv[2])
    elif ('prepare' == sys.argv[1]):
        prepare_json(sys.argv[2], sys.argv[3] if sys.argv[3] != "none" else None,
                                  sys.argv[4] if sys.argv[4] != "none" else None,
                                  sys.argv[5] if sys.argv[5] != "none" else None)
    else:
        print('Undefined operation')
        fns = ["./pool-219480.json", "./pool-221476.json", "./pool-221655.json", "./pool-232727.json", "./pool-232861.json", "./pool-234832.json", "./pool-236127.json", "./pool-236165.json", "./pool-241589.json"]
        classes = []
        for fn in fns:
            answers, sign_ids = stat_file("/home/vbystricky/datasets/traffic_signs/toloka/temp/json/{}".format(fn))
            for sign in sign_ids:
                if (not sign in classes):
                    classes += [sign]
        for cl in sorted(classes):
            print(cl)

