<jupyter_start><jupyter_text>Fix OCR'd text in latitude and longitude stamps<jupyter_code>#print a 10-sample of key-value pairs from bad_lats
{k: v for i, (k,v) in enumerate(dict((k, bad_lats[k]) for k in bad_lats.keys()).items()) if i < 10}
#make a shallow copy of bad_lats dictionary
bad_lats_copy = bad_lats.copy()
#remove all frames from bad_lats where number of digits in latitude value is not 7
bad_lats_copy = {k:v for (k,v) in bad_lats_copy.items() if sum(c.isdigit() for c in v) == 7}
len(bad_lats_copy)
#remove all spaces from values
bad_lats_copy = {k: v.replace(" ","") for (k,v) in bad_lats_copy.items()}
#check all existing positions of a period in values
pos_of_period = {}
for v in bad_lats_copy.values():
    pos_list = [pos for pos, char in enumerate(v) if char == '.']
    if tuple(pos_list) not in pos_of_period:
        pos_of_period[tuple(pos_list)] = 1
    else:
        pos_of_period[tuple(pos_list)] += 1
pos_of_period
sum(pos_of_period.values())
#check latitude stamps that don't have periods
for v in bad_lats_copy.values():
    if '.' not in v:
        print(v)
#print a 10-sample of key-value pairs from bad_lats_copy
{k: v for i, (k,v) in enumerate(dict((k, bad_lats_copy[k]) for k in bad_lats_copy.keys()).items()) if i < 10}
#remove all frames where position of period is anything other than (6,) or (6,8), or there is no period at all
bad_lats_copy = {k: v for (k,v) in bad_lats_copy.items() if tuple([pos for pos, char in enumerate(v) if char == '.'])
    in ((6,),(6,8),())}
len(bad_lats_copy)
#function to remove trailing periods, to insert period for stamps that need it, to insert spaces,
#and to convert text to float
def lat_clean_transform(stamp):
    no_trail_prd = stamp[:8]
    if '.' not in no_trail_prd:
        stamp_new = no_trail_prd[:2] + ' ' + no_trail_prd[2:4] + ' ' + no_trail_prd[4:6] + '.' + no_trail_prd[6:6]
    else:
        stamp_new = no_trail_prd[:2] + ' ' + no_trail_prd[2:4] + ' ' + no_trail_prd[4:8]
    return float(stamp_new.split(' ')[0]) + float(stamp_new.split(' ')[1])/60 + \
                     float(stamp_new.split(' ')[2])/3600
replaced_lats = {k: lat_clean_transform(v) for (k,v) in bad_lats_copy.items()}
len(replaced_lats)
#print a 10-sample of key-value pairs from replaced_lats
{k: v for i, (k,v) in enumerate(dict((k, replaced_lats[k]) for k in replaced_lats.keys()).items()) if i < 10}
sum(x is None for x in replaced_lats.values())
#print a 10-sample of key-value pairs from bad_lons
{k: v for i, (k,v) in enumerate(dict((k, bad_lons[k]) for k in bad_lons.keys()).items()) if i < 10}
#make a shallow copy of bad_lons dictionary
bad_lons_copy = bad_lons.copy()
#remove all frames from bad_lons where number of digits in longitude value is not 7 or 8 with first digit being 1
bad_lons_copy = {k:v for (k,v) in bad_lons_copy.items() if (sum(c.isdigit() for c in v) == 7) |
                ((sum(c.isdigit() for c in v) == 8) & (v.startswith('1')))}
len(bad_lons_copy)
#remove all spaces from values
bad_lons_copy = {k: v.replace(" ","") for (k,v) in bad_lons_copy.items()}
#print a 10-sample of key-value pairs from bad_lons_copy
{k: v for i, (k,v) in enumerate(dict((k, bad_lons_copy[k]) for k in bad_lons_copy.keys()).items()) if i < 10}
#check all existing positions of a period in values
pos_of_period = {}
for v in bad_lons_copy.values():
    pos_list = [pos for pos, char in enumerate(v) if char == '.']
    if tuple(pos_list) not in pos_of_period:
        pos_of_period[tuple(pos_list)] = 1
    else:
        pos_of_period[tuple(pos_list)] += 1
pos_of_period
sum(pos_of_period.values())
#check longitude stamps that don't have periods
for v in bad_lons_copy.values():
    if '.' not in v:
        print(v)
#remove all frames where position of period is confusing
bad_lons_copy = {k: v for (k,v) in bad_lons_copy.items() if tuple([pos for pos, char in enumerate(v) if char == '.'])
    in ((7,),(6,),(7,9),(6,8),())}
len(bad_lons_copy)
#print a 10-sample of key-value pairs from bad_lons_copy
{k: v for i, (k,v) in enumerate(dict((k, bad_lons_copy[k]) for k in bad_lons_copy.keys()).items()) if i < 10}
#len({k: v for (k,v) in bad_lons_copy.items() if len(v) < 7})
#function to remove trailing periods, to insert period for stamps that need it, to insert spaces,
#and to convert text to float
def lon_clean_transform(stamp):
    if (sum(c.isdigit() for c in stamp) == 7) & (tuple([pos for pos, char in enumerate(stamp) if char == '.']) in ((6,),(6,8),())):
        no_trail_prd = stamp[:8]
        if '.' not in no_trail_prd:
            stamp_new = no_trail_prd[:2] + ' ' + no_trail_prd[2:4] + ' ' + no_trail_prd[4:6] + '.' + no_trail_prd[6:6]
        else:
            stamp_new = no_trail_prd[:2] + ' ' + no_trail_prd[2:4] + ' ' + no_trail_prd[4:8]
        return float(stamp_new.split(' ')[0]) + float(stamp_new.split(' ')[1])/60 + \
                         float(stamp_new.split(' ')[2])/3600
    elif (sum(c.isdigit() for c in stamp) == 8) & (tuple([pos for pos, char in enumerate(stamp) if char == '.']) in ((7,),(7,9),())):
        no_trail_prd = stamp[:9]
        if '.' not in no_trail_prd:
            stamp_new = no_trail_prd[:3] + ' ' + no_trail_prd[3:5] + ' ' + no_trail_prd[5:7] + '.' + no_trail_prd[7:7]
        else:
            stamp_new = no_trail_prd[:3] + ' ' + no_trail_prd[3:5] + ' ' + no_trail_prd[5:9]
        return float(stamp_new.split(' ')[0]) + float(stamp_new.split(' ')[1])/60 + \
                         float(stamp_new.split(' ')[2])/3600
replaced_lons = {k: lon_clean_transform(v) for (k,v) in bad_lons_copy.items()}
len(replaced_lons)
#print a 10-sample of key-value pairs from replaced_lons
{k: v for i, (k,v) in enumerate(dict((k, replaced_lons[k]) for k in replaced_lons.keys()).items()) if i < 10}
sum(x is None for x in replaced_lons.values())
#bad_lons_copy['frame28907']
replaced_lats = {k: {'Lat':v} for (k,v) in replaced_lats.items()}
replaced_lons = {k: {'Lon':v} for (k,v) in replaced_lons.items()}
#print a 10-sample of key-value pairs from replaced_lats
{k: v for i, (k,v) in enumerate(dict((k, replaced_lats[k]) for k in replaced_lats.keys()).items()) if i < 10}
#print a 10-sample of key-value pairs from replaced_lons
{k: v for i, (k,v) in enumerate(dict((k, replaced_lons[k]) for k in replaced_lons.keys()).items()) if i < 10}
sum(x == {'Lat': None} for x in replaced_lats.values())
sum(x == {'Lon': None} for x in replaced_lons.values())
len(replaced_lats.keys() & replaced_lons.keys())
len(replaced_lats.keys() - replaced_lons.keys())
len(replaced_lons.keys() - replaced_lats.keys())
image_dict_final = copy.deepcopy(image_dict_rstr)
#number of keys in value dictionary in image_dict_final
keys_0 = 0
keys_1 = 0
keys_2 = 0
for v in image_dict_final.values():
    n_keys = len(v.keys())
    if n_keys == 0:
        keys_0 += 1
    elif n_keys == 1:
        keys_1 += 1
    elif n_keys == 2:
        keys_2 += 2
print("0 keys:",keys_0,"\n1 key:",keys_1,"\n2 keys:",keys_2)
#number of keys in value dictionary in replaced_lons
keys_0 = 0
keys_1 = 0
keys_2 = 0
for v in replaced_lons.values():
    n_keys = len(v.keys())
    if n_keys == 0:
        keys_0 += 1
    elif n_keys == 1:
        keys_1 += 1
    elif n_keys == 2:
        keys_2 += 2
print("0 keys:",keys_0,"\n1 key:",keys_1,"\n2 keys:",keys_2)
#print a 10-sample of key-value pairs from image_dict_final
{k: v for i, (k,v) in enumerate(dict((k, image_dict_final[k]) for k in image_dict_final.keys()).items()) if i < 10}
#check how many frames have no latitude or longitude value in image_dict_final before fixed values are inserted
lon_none = 0
lat_none = 0
lat_lon_none = 0
for v in image_dict_final.values():
    if (v['Lat'] is None) & (v['Lon'] is None):
        lat_lon_none += 1
    elif v['Lat'] is None:
        lat_none += 1
    elif v['Lon'] is None:
        lon_none += 1
print("Frames without Lat:",lat_none,"\nFrames without Lon:",lon_none,"\nFrames without both:",lat_lon_none)
#copy fixed latitude and longitude stamps into final image_dict_final dictionary
for k, v in image_dict_final.items():
    if (v['Lat'] is None) & (k in replaced_lats):
        image_dict_final[k]['Lat'] = replaced_lats[k]['Lat']
    if (v['Lon'] is None) & (k in replaced_lons):
        image_dict_final[k]['Lon'] = replaced_lons[k]['Lon']
#check how many frames ended up having no latitude or longitude value
lon_none = 0
lat_none = 0
lat_lon_none = 0
for v in image_dict_final.values():
    if (v['Lat'] is None) & (v['Lon'] is None):
        lat_lon_none += 1
    elif v['Lat'] is None:
        lat_none += 1
    elif v['Lon'] is None:
        lon_none += 1
print("Frames without Lat:",lat_none,"\nFrames without Lon:",lon_none,"\nFrames without both:",lat_lon_none)
len(image_dict_final)
#remove frames without latitude and/or longitude value
for k in list(image_dict_final.keys()):
    if (image_dict_final[k]['Lat'] is None) | (image_dict_final[k]['Lon'] is None):
        del image_dict_final[k]
len(image_dict_final)
#check how many remaining latitude and longitude values fall outside the expected bounds
lat_out_of_bounds = 0
lon_out_of_bounds = 0
both_out_of_bounds = 0
for v in image_dict_final.values():
    if ((v['Lat'] < 37.0) | (v['Lat'] > 47.7)) & ((v['Lon'] < 77.0) | (v['Lon'] > 122.33)):
        both_out_of_bounds += 1
    elif (v['Lat'] < 37.0) | (v['Lat'] > 47.7):
        lat_out_of_bounds += 1
    elif (v['Lon'] < 77.0) | (v['Lon'] > 122.33):
        lon_out_of_bounds += 1
print("Both of out bounds:",both_out_of_bounds,"\nLat out of bounds:",lat_out_of_bounds,"\nLon out of bounds:",lon_out_of_bounds)
#remove these frames that have latitude and longitude values fall outside the expected bounds
for k in list(image_dict_final.keys()):
    if ((image_dict_final[k]['Lat'] < 37.0) | (image_dict_final[k]['Lat'] > 47.7) | 
        (image_dict_final[k]['Lon'] < 77.0) | (image_dict_final[k]['Lon'] > 122.33)):
        del image_dict_final[k]
len(image_dict_final)
#print a 10-sample of key-value pairs from image_dict_final
{k: v for i, (k,v) in enumerate(dict((k, image_dict_final[k]) for k in image_dict_final.keys()).items()) if i < 10}
json_txt = json.dumps(image_dict_final)
f = open("image_txt.json","w")
f.write(json_txt)
f.close()
#print a 10-sample of key-value pairs from replaced_lons for keys that are also found in replaced_lats dictionary
{k: v for i, (k,v) in enumerate(dict((k, replaced_lons[k]) for k in (image_dict.keys() & bad_lats.keys())).items()) if i < 10}
#from https://stackoverflow.com/a/7205107/9987623
#function to perform nested merge of dictionaries
def merge_dicts(a, b, path=None):
    "merges b into a"
    if path is None: path = []
    for key in b:
        if key in a:
            if isinstance(a[key], dict) and isinstance(b[key], dict):
                merge(a[key], b[key], path + [str(key)])
            elif a[key] == b[key]:
                pass # same leaf value
            else:
                raise Exception('Conflict at %s' % '.'.join(path + [str(key)]))
        else:
            a[key] = b[key]
    return a
dict({k_lat: v_lat.update({'Lon': None}) for (k_lat,v_lat) in replaced_lats.items() if k_lat in (replaced_lats.keys() - 
    replaced_lons.keys())}.items() + {k_lon: v_lon.update({'Lat': None}) for (k_lon,v_lon) in 
    replaced_lons.items() if k_lon in (replaced_lons.keys() - replaced_lats.keys())}.items() + \
    {k_lat_lon: {'Lat': replaced_lats[k_lat_lon]['Lat'], 'Lon': replaced_lons[k_lat_lon]['Lon']} for k_lat_lon in 
     (replaced_lats.keys() & replaced_lons.keys())}.items())
#print a 10-sample of key-value pairs from replaced_lats
{k: v for i, (k,v) in enumerate(dict((k, replaced_lats[k]) for k in replaced_lats.keys()).items()) if i < 10}
{k_lat: v_lat.update({'Lon': None}) for (k_lat,v_lat) in replaced_lats.items() if k_lat in (replaced_lats.keys() - 
    replaced_lons.keys())}
#this probably modified the original replaced_lats dictionary
{k_lat: v_lat.update({'Lon': None}) for (k_lat,v_lat) in replaced_lats.items()}
lats_not_lons = {k_lat: v_lat for (k_lat,v_lat) in replaced_lats.items() if k_lat 
                 in (replaced_lats.keys() - replaced_lons.keys())}
{k_lon: v_lon.update({'Lat': None}) for (k_lon,v_lon) in replaced_lons.items()}
lons_not_lats = {k_lon: v_lon for (k_lon,v_lon) in replaced_lons.items() if k_lon 
                 in (replaced_lons.keys() - replaced_lats.keys())}
lats_and_lons = {k_lat_lon: {'Lat': replaced_lats[k_lat_lon]['Lat'], 'Lon': replaced_lons[k_lat_lon]['Lon']} 
                 for k_lat_lon in (replaced_lats.keys() & replaced_lons.keys())}
dicts_to_merge = [lats_not_lons, lons_not_lats, lats_and_lons]
merged_replacements = dict(ChainMap(*dicts_to_merge))

# merged_replacements = {**{k_lat: v_lat.update({'Lon': None}) for (k_lat,v_lat) in replaced_lats.items() if k_lat in (replaced_lats.keys() - 
#     replaced_lons.keys())}, **{k_lon: v_lon.update({'Lat': None}) for (k_lon,v_lon) in 
#     replaced_lons.items() if k_lon in (replaced_lons.keys() - replaced_lats.keys())},
#     **{k_lat_lon: {'Lat': replaced_lats[k_lat_lon]['Lat'], 'Lon': replaced_lons[k_lat_lon]['Lon']} for k_lat_lon in 
#      (replaced_lats.keys() & replaced_lons.keys())}}
len(lats_and_lons)
len(lats_not_lons)
len(replaced_lats.keys() - replaced_lons.keys())
#print a 10-sample of key-value pairs from lats_not_lons
{k: v for i, (k,v) in enumerate(dict((k, lats_not_lons[k]) for k in lats_not_lons.keys()).items()) if i < 10}
merged_replacements
len(merged_replacements)
sum(x is None for x in merged_replacements.values())
{merge_dicts(copy.deepcopy(replaced_lats),copy.deepcopy(replaced_lons)) for replaced_lats}

merge_dicts(__, __) for dict(replaced_lats.items()
{k_lat_lon: {'Lat': replaced_lats[k_lat_lon]['Lat'], 'Lon': replaced_lons[k_lat_lon]['Lon']} for k_lat_lon in replaced_lats.key() }
'''
{'frame8359': {'Lat': None, 'Lon': 85.89155555555556},
 'frame6845': {'Lat': None, 'Lon': 84.04952777777778},
 'frame7862': {'Lat': None, 'Lon': 85.25738888888888},
 'frame32668': {'Lat': None, 'Lon': 115.90894444444444},
 'frame467': {'Lat': None, 'Lon': 77.46622222222223},
 'frame36963': {'Lat': None, 'Lon': None},
 'frame24656': {'Lat': None, 'Lon': 107.78122222222223},
 'frame6392': {'Lat': None, 'Lon': 83.43738888888889},
 'frame9776': {'Lat': None, 'Lon': 87.869},
 'frame5612': {'Lat': None, 'Lon': 82.36247222222222}}
'''
a_test = {'a':'foo','b':'bar','c':'baz'}
b_test = {'a':'spam','c':'ham','x':'blah'}

r_test = dict(a_test.items() + b_test.items())<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16131938188621717, 'fraction_numerical': 0.03139075601136442, 'mean_word_length': 4.200720720720721, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 11, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 8, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '514265', 'license': 'MIT', 'n_tokens_mistral': 5908, 'n_tokens_neox': 5553, 'n_words': 1469, 'path': 'GPS_Text_Recognition.ipynb', 'repo_name': 'sasha-kap/CV-to-Maps', 'size': '14431'}