<jupyter_start><jupyter_text>Working df ready. Now infer location.<jupyter_code>locs=df[['category','icategory','tags','title','content']]
locs
import json
lak=json.loads(open('geo/huncoords.json','r').read())
megyek=json.loads(open('geo/megyek.json','r').read())
laks=[i for i in megyek.keys() if ((megyek[i]!='Más megye')and(i in lak.keys()))]
def levenshteinDistance(s1, s2):
    if len(s1) > len(s2):
        s1, s2 = s2, s1

    distances = range(len(s1) + 1)
    for i2, c2 in enumerate(s2):
        distances_ = [i2+1]
        for i1, c1 in enumerate(s1):
            if c1 == c2:
                distances_.append(distances[i1])
            else:
                distances_.append(1 + min((distances[i1], distances[i1 + 1], distances_[-1])))
        distances = distances_
    return distances[-1]
#look for perfect matches
decoded_locs={}
min_loc_len=5
for i in locs.index:
    if '[' in locs.loc[i]['tags']:
        k=locs.loc[i]['tags'].replace("'",'"')
        for j in json.loads(k):
            if j in laks:
                if i not in decoded_locs:decoded_locs[i]=[]
                decoded_locs[i].append(j)
                print('1',k,j)
    for j in locs.loc[i]['title'].split():
        if j in laks:
            if i not in decoded_locs:decoded_locs[i]=[]
            decoded_locs[i].append(j)
            print('2',j)
    for j in locs.loc[i]['content'].split():
        if j in laks:
            if i not in decoded_locs:decoded_locs[i]=[]
            decoded_locs[i].append(j)
            print('3',j)
    for j in locs.loc[i]['title'].split():
        for k in laks:
            if ((len(j)>min_loc_len) and (len(k)>min_loc_len)):
                if levenshteinDistance(j,k)<3:
                    if i not in decoded_locs:decoded_locs[i]=[]
                    decoded_locs[i].append(k)
                    print('4',j,k)
    for j in locs.loc[i]['content'].split():
        for k in laks:
            if ((len(j)>min_loc_len) and (len(k)>min_loc_len)):
                if levenshteinDistance(j,k)<3:
                    if i not in decoded_locs:decoded_locs[i]=[]
                    decoded_locs[i].append(k)
                    print('5',j,k)
predicted_locs=[]
for i in range(len(locs.index)):
    if locs.index[i] in decoded_locs: 
        dummy=list(set(decoded_locs[locs.index[i]]))
        if len(dummy)==1:
            dummy=dummy[0]
        else:
            dummy=repr(list(set(decoded_locs[locs.index[i]])))
        predicted_locs.append(dummy)
    else: 
        predicted_locs.append('')
locs['predicted_locs']=predicted_locs
locs['actual_locs']=predicted_locs
lm=locs[['title','content','predicted_locs','actual_locs']]
lm.to_excel('data/szekelyhon_locs.xlsx')
lm.to_excel('data/szekelyhon_locs_curated.xlsx')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1608492263404102, 'fraction_numerical': 0.013314141777617848, 'mean_word_length': 2.49685534591195, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '277904', 'license': 'MIT', 'n_tokens_mistral': 1056, 'n_tokens_neox': 1034, 'n_words': 184, 'path': '.ipynb_checkpoints/2_location_extractor-checkpoint.ipynb', 'repo_name': 'szekelydata/medveterkep', 'size': '2779'}