<jupyter_start><jupyter_text>Fuzzy search<jupyter_code>import codecs, difflib, Levenshtein
import jellyfish,textdistance


str_match ='हनुमान'
distance_df = pd.DataFrame(columns=["word","diffl","Levenshtein","Sorensen","Jaccard","fuzz_ratio","partial_ratio",
                                    "token_sort_ratio","token_set_ratio"])
for word in tqdm(final_text):
    diffl   = difflib.SequenceMatcher(None, str_match, word).ratio()
    lev     = Levenshtein.ratio(str_match, word) 
    sor     =  textdistance.sorensen(str_match, word)
    jac     =  textdistance.jaccard(str_match, word)
    fuzz_ratio = fuzz.ratio(str_match,word)/100 
    partial_ratio = fuzz.partial_ratio(str_match,word)/100
    token_sort_ratio = fuzz.token_sort_ratio(str_match,word)/100
    token_set_ratio = fuzz.token_set_ratio(str_match,word)/100
    
    
    distance_list = [word,diffl,lev,sor,jac,fuzz_ratio,partial_ratio,token_sort_ratio,token_set_ratio]
    distance_series = pd.Series(dict(zip(distance_df.columns, distance_list)))
    
    distance_df = distance_df.append(distance_series, ignore_index=True)
#     print( diffl, lev, sor, jac)
distance_df[distance_df["Levenshtein"]>0.80]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11867219917012448, 'fraction_numerical': 0.012448132780082987, 'mean_word_length': 5.0, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '597645', 'license': 'MIT', 'n_tokens_mistral': 457, 'n_tokens_neox': 434, 'n_words': 51, 'path': 'src/ramayanaocr/notebook/fuzzy-test.ipynb', 'repo_name': 'Sudipt-panda/ramayanaocr', 'size': '1205'}