<jupyter_start><jupyter_text>Spell Correction<jupyter_code>import sys
# !{sys.executable} -m pip install fuzzywuzzy
# alternative for 4-10x faster computation: 
!{sys.executable} -m pip install fuzzywuzzy[speedup]
from fuzzywuzzy import fuzz
fuzz.ratio("Electronic City Phase One", "Electronic City Phase One, Bangalore")
fuzz.partial_ratio("Electronic City Phase One", "Electronic City Phase One, Bangalore")
fuzz.ratio('Narendra Modi', 'Narendra D. Modi')
fuzz.partial_ratio('Narendra Modi', 'Narendra D. Modi')
fuzz.token_sort_ratio('<NAME>', '<NAME>')
fuzz.token_set_ratio('<NAME>', '<NAME>')
from fuzzywuzzy import process
query = 'Gujrat'
choices = ['Gujarat', 'Gujjar', 'Gujarat Govt.']
# Get a list of matches ordered by score, default limit to 5
print(process.extract(query, choices))

# If we want only the top one
process.extractOne(query, choices)
query = 'Banglore'
choices = ['Bangalore', 'Bengaluru']
print(process.extract(query, choices))
process.extractOne(query, choices)
# Let's take an example of a common search typo in online shopping:
query = 'chili'
choices = ['chilli', 'chilled', 'chilling']
print(process.extract(query, choices))
process.extractOne(query, choices)<jupyter_output>[('chilli', 91), ('chilling', 77), ('chilled', 67)]<jupyter_text>Jellyfish<jupyter_code>import sys
!{sys.executable} -m pip install jellyfish
import jellyfish
correct_example = ('Narendra Modi', 'Narendra Modi')
damodardas_example = ('Narendra Modi', '<NAME>')
modi_typo_example = ('Narend<NAME>', '<NAME>')
gujarat_typo_example = ('Gujarat', 'Gujrat')

examples = [correct_example, damodardas_example, modi_typo_example, gujarat_typo_example]
def calculate_distance(function, examples=examples):
    for ele in examples:
        print(f'{ele}: {function(*ele)}') 
calculate_distance(jellyfish.levenshtein_distance)
calculate_distance(jellyfish.damerau_levenshtein_distance)
calculate_distance(jellyfish.hamming_distance)
calculate_distance(jellyfish.jaro_distance) 
calculate_distance(jellyfish.jaro_winkler)<jupyter_output>('<NAME>', '<NAME>'): 1.0
('<NAME>', '<NAME>'): 0.9625
('<NAME>', '<NAME>'): 0.9846153846153847
('Gujarat', 'Gujrat'): 0.9277777777777778<jupyter_text>Phonetic Word Similarity What is a phonetic encoding?<jupyter_code>jellyfish.soundex('Jellyfish')
jellyfish.nysiis('Jellyfish')
jellyfish.metaphone('Jellyfish')
jellyfish.match_rating_codex('Jellyfish')<jupyter_output><empty_output><jupyter_text>Metaphone + Levenshtein<jupyter_code>jellyfish.levenshtein_distance(jellyfish.metaphone('write'), jellyfish.metaphone('right'))
examples+= [('write', 'right'), ('Mangalore', 'Bangalore'), ('Delhi', 'Dilli')] # adding a few examples to show how cool this is
def calculate_phonetic_distance(phonetic_func, distance_func, examples=examples):
    print("Word\t\tSound\t\tWord\t\t\tSound\t\tPhonetic Distance")
    for ele in examples:
        correct, typo = ele[0], ele[1]
        phonetic_correct, phonetic_typo = phonetic_func(correct), phonetic_func(typo)
        phonetic_distance = distance_func(phonetic_correct, phonetic_typo)
        print(f'{correct:<10}\t{phonetic_correct:<10}\t{typo:<20}\t{phonetic_typo:<10}\t{phonetic_distance:<10}') 
        
calculate_phonetic_distance(phonetic_func=jellyfish.metaphone, distance_func=jellyfish.levenshtein_distance)<jupyter_output>Word		Sound		Word			Sound		Phonetic Distance
Narendra Modi	NRNTR MT  	Narendra Modi       	NRNTR MT  	0         
Narendra Modi	NRNTR MT  	Narendra D. Modi    	NRNTR T MT	2         
Narendra Modi	NRNTR MT  	Narendar Modi       	NRNTR MT  	0         
Gujarat   	KJRT      	Gujrat              	KJRT      	0         
write     	RT        	right               	RT        	0         
Mangalore 	MNKLR     	Bangalore           	BNKLR     	1         
Delhi     	TLH       	Dilli               	TL        	1<jupyter_text>American Soundex<jupyter_code>calculate_phonetic_distance(phonetic_func=jellyfish.soundex, distance_func=jellyfish.levenshtein_distance)<jupyter_output>Word		Sound		Word			Sound		Phonetic Distance
Narendra Modi	N653      	Narendra Modi       	N653      	0         
Narendra Modi	N653      	Narendra D. Modi    	N653      	0         
Narendra Modi	N653      	Narendar Modi       	N653      	0         
Gujarat   	G263      	Gujrat              	G263      	0         
write     	W630      	right               	R230      	2         
Mangalore 	M524      	Bangalore           	B524      	1         
Delhi     	D400      	Dilli               	D400      	0<jupyter_text>Updating the Original Corpus with FlashText<jupyter_code>import sys
!{sys.executable} -m pip install flashtext
from flashtext.keyword import KeywordProcessor
keyword_processor = KeywordProcessor()
keyword_processor.add_keyword('Delhi', 'NCR') # notice we are adding tuples here
keyword_processor.add_keyword('Bombay', 'Mumbai')
keywords_found = keyword_processor.extract_keywords('I love the food in Delhi and the people in Bombay')
keywords_found
# ['NCR', 'Mumbai']<jupyter_output><empty_output><jupyter_text>How about we replace them now?<jupyter_code>from flashtext.keyword import KeywordProcessor
keyword_processor = KeywordProcessor()
keyword_processor.add_keyword('Delhi', 'NCR')
keyword_processor.add_keyword('Bombay', 'Mumbai')
replaced_sentence = keyword_processor.replace_keywords('I love the food in Delhi and the people in Bombay')
replaced_sentence
# 'I love the food in NCR and the people in Mumbai'<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10441988950276243, 'fraction_numerical': 0.021915285451197055, 'mean_word_length': 4.3560157790927025, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '311721', 'license': 'MIT', 'n_tokens_mistral': 2052, 'n_tokens_neox': 1851, 'n_words': 413, 'path': 'Part-02-B.ipynb', 'repo_name': 'sarsiz/nlp-python-deep-learning', 'size': '5527'}