<jupyter_start><jupyter_text><jupyter_code>import re #regular expression modules
import string
from collections import Counter
import numpy as np
#tokenization
def read_corpus(filename):
  with open(filename, "r") as file:
    lines = file.readlines()
    words = [] 
    for line in lines:
      words += re.findall(r'\w+', line.lower()) # "w+" This expression matches the alphanumeric character in the string

  return words
words = read_corpus("./t8.shakespeare.txt") #read file
print(len(words)) #total words in the corpus
vocabs = set(words)
print(len(vocabs)) #unique words in the vocabulary
word_counts = Counter(words) #word_counts became integer
print(word_counts) #classification count words
print(word_counts["boy"]) #sample "boy" total words count
print(type(word_counts))
print(type(words))
total_word_count = float(sum(word_counts.values())) #convert to float for probabilities
print(total_word_count)
word_probas = {word: word_counts[word] / total_word_count for word in word_counts.keys()} #Using Dictionary Comprehension
print(word_probas["boy"])
def split(word):
  return [(word[:i], word[i:]) for i in range(len(word) + 1)]
print(split("trash"))
#DELETE (remove a letter)
def delete(word):
  return [l + r[1:] for l,r in split(word) if r] 

# "l" display ['', 't', 'tr', 'tra', 'tras']
# "r[1:]" display ['rash', 'ash', 'sh', 'h', '']
print(delete("trash"))
#SWAP (swap 2 adjacent letters)
def swap(word):
  return [l + r[1] + r[0] + r[2:] for l, r in split(word) if len(r)>1]
print(swap("trash"))
string.ascii_lowercase
#REPLACE (change one letter to another)
def replace(word):
  letters = string.ascii_lowercase
  return [l + c + r[1:] for l, r in split(word) if r for c in letters]

# "l" display ['', 't', 'tr', 'tra'] over all ascii_lowercase
# "c" display all ascii_lowercase ordered
# "r[1:]" display the rest of the letter
print(replace("trash"))
#INSERT (add a letter)
def insert(word):
  letters = string.ascii_lowercase
  return [l + c + r for l, r in split(word) for c in letters]
print(insert("trash"))
def edit1(word):
  return set(delete(word) + swap(word) + replace(word) + insert(word))
print(edit1("trash"))
#Performaing level_1 editing twice
def edit2(word):
  return set(e2 for e1 in edit1(word) for e2 in edit1(e1))
print(edit2("trash"))
def correct_spelling(word, vocabulary, word_probabilities):
  if word in vocabulary:
    print(f"{word} is already correctly spelt")
    return 

  suggestions = edit1(word) or edit2(word) or [word]
  best_guesses = [w for w in suggestions if w in vocabulary]
  return [(w, word_probabilities[w]) for w in best_guesses]
word = "femily"
corrections = correct_spelling(word, vocabs, word_probas)

if corrections:
  print(corrections)
  probs = np.array([c[1] for c in corrections])
  best_ix = np.argmax(probs)
  correct = corrections[best_ix][0]
  print(f"{correct} is suggested for {word}")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1419710544452102, 'fraction_numerical': 0.00895933838731909, 'mean_word_length': 5.494407158836689, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '411793', 'license': 'MIT', 'n_tokens_mistral': 1029, 'n_tokens_neox': 1001, 'n_words': 314, 'path': 'spell_checker_probabilistic_autocorrection_model.ipynb', 'repo_name': 'Checkmate-2/Spelling_Detection_and_Correction', 'size': '2902'}