<jupyter_start><jupyter_text>unicode descriptions<jupyter_code>chars_desc_replacements={}
notfound = []

for k in chars_counter.keys():
    url = 'https://www.fileformat.info/info/unicode/char/search.htm?q='+k+'+&preview=entity'
    html_soup = BeautifulSoup(get(url).text, 'html.parser')
    try:
        tab = html_soup.find_all('table', {'class': 'table table-list table-striped'})[0]
        desc = tab.find_all('tr' , {'class':"row0"})[0].find_all('td')[2].text
        chars_desc_replacements[k] = [desc , '']
    except IndexError:
        notfound.append(k)

notfound
for k in chars_desc_replacements.keys():
    chars_desc_replacements[k][1] = k<jupyter_output><empty_output><jupyter_text>Char replacements<jupyter_code>chars_desc_replacements = json.load(open("../char_replacements.dict"))<jupyter_output><empty_output><jupyter_text>Cleaning dataset<jupyter_code>with open(LOCAL_PATH) as f:
    poets = json.load(f)
duple = copy.deepcopy(poets)
temp = { x[0] : x[1][1]  for x in chars_desc_replacements.items()}

# flag=5
for p in poets.keys():
    print (p , poets[p]['name_fa']) 
    
    for w_i in range(len(poets[p]['works'])): 
        duple[p]['works'][w_i][1] = []
        for p_i, poem in enumerate(poets[p]['works'][w_i][1]):
#             duple[p]['works'][w_i][1].append([poem, []])
            s = re.sub(r'.', lambda m: temp.get(m.group()), poem)
            duple[p]['works'][w_i][1].append(s)
#     if flag:
#         flag -= 1
#     else:
#         break
    
del temp


with open(LOCAL_PATH,'w') as f:
    json.dump(duple,f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18141592920353983, 'fraction_numerical': 0.008849557522123894, 'mean_word_length': 3.9779874213836477, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '638945', 'license': 'MIT', 'n_tokens_mistral': 626, 'n_tokens_neox': 610, 'n_words': 90, 'path': 'cleaning_and_standardization/suppl_cleaning.ipynb', 'repo_name': 'pithysr/persian-poetry', 'size': '1582'}