<jupyter_start><jupyter_text>Vectorize spell checking engine DescriptionImplementation of a fast spell checking and suggestion engine using character vectors. Methodology:- Acquire a words list that will serve as the absolute truth of spellings.- Use tfidf or phoenetic vectorization method to encode words.- Use similarity metrics to identify possible respelling of out of vocabulary terms in a document. Word list resources- http://wordlist.aspell.net/- https://github.com/dwyl/english-words :: https://github.com/dwyl/english-words/blob/master/words_dictionary.json<jupyter_code># -*- coding: utf-8 -*-
%load_ext autotime
import os
import re
import numpy as np
import pandas as pd
import nltk
import json

from nltk.corpus import words
from joblib import Parallel, delayed
import multiprocessing as mp
import multiprocessing

from collections import Counter

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from nltk.metrics.distance import edit_distance

from scipy.stats import rankdata

import spacy

from contexttimer import Timer
import requests
from lsh import minhash, cache
from sklearn.decomposition import TruncatedSVD
from nltk import word_tokenize
from enchant.checker import SpellChecker
from enchant import Dict
class OptimizedSpellChecker(SpellChecker):
    '''
    Reduces the tokens only to unique words in the text. Output is not in the same order relative
    to the original text.
    '''
    dict_words = set()
    
    def __init__(self, lang=None, text=None, tokenize=None, chunkers=None, filters=None):
        super().__init__(
            lang=lang, text=text, tokenize=tokenize, chunkers=chunkers, filters=filters
        )
        
    def set_tokens(self, tokens):
        """Set the text to be spell-checked.
        This method must be called, or the 'text' argument supplied
        to the constructor, before calling the 'next()' method.
        """
        self._tokens = enumerate(tokens)
        
    def next(self):
        """Process text up to the next spelling error.
        
        This method is designed to support the iterator protocol.
        Each time it is called, it will advance the 'word' attribute
        to the next spelling error in the text.  When no more errors
        are found, it will raise StopIteration.
        
        The method will always return self, so that it can be used
        sensibly in common idioms such as:
            for err in checker:
                err.do_something()
        
        """
        # Find the next spelling error.
        # The uncaught StopIteration from next(self._tokens)
        # will provide the StopIteration for this method
        while True:
            pos, word = next(self._tokens)
            if word in self.dict_words:
                continue
            if self.dict.check(word):
                self.dict_words.add(word)
                continue
            if word in self._ignore_words:
                continue
            self.word = word
            self.wordpos = pos
            if word in self._replace_words:
                self.replace(self._replace_words[word])
                continue
            break
        return self

en_dict = Dict(u'en_US')
spellchecker = OptimizedSpellChecker(u"en_US")
%time en_dict.suggest(u'nighttime')<jupyter_output>CPU times: user 32.7 ms, sys: 319 µs, total: 33 ms
Wall time: 32.2 ms<jupyter_text>![im](https://cdn-images-1.medium.com/max/1600/1*5bvgtaEv3UcowA3vw8SWpQ.png)<jupyter_code>import itertools
with open('./data/words.txt') as fl:
    words = dict(enumerate([i.strip() for i in fl.readlines() if i.strip().isalpha()]))

print(len(words))
hasher = minhash.MinHasher(seeds=100, char_ngram=3, hashbytes=4)
lsh_cache = cache.Cache(bands=10, hasher=hasher)

for word_id, word in words.items():
    lsh_cache.add_fingerprint(hasher.fingerprint(word), word_id)

duplicate_pairs = set()

total_bins = 0

for cache_bin in lsh_cache.bins:
    for bucket_id in cache_bin:
        if len(cache_bin[bucket_id]) > 1:
            pairs = set(itertools.combinations(cache_bin[bucket_id], r=2))
            duplicate_pairs.update(pairs)
        total_bins += 1

duplicate_pairs = list(duplicate_pairs)
total_bins
total_bins
[(words[i], words[j]) for i, j in duplicate_pairs[:10]]
swords = set(words.values())
'Python' in swords
tfidf = TfidfVectorizer(analyzer='char', ngram_range=(3,3), dtype=np.float32)
tsvd = TruncatedSVD(n_components=50)

dict_words_vecs = tfidf.fit_transform(words)
dict_words_vecs = tsvd.fit_transform(dict_words_vecs)
w = 'I pulled out the words into a simple new-line-delimited text file. Which is more useful when building apps or importing into databases etc.'
tokens = word_tokenize(w)
def load_text(doc_name='11758940.txt', path='/home/avsolatorio/WBG/NLP/WB/CORPUS/RAW/eap_files'):
    fname = os.path.join(path, doc_name)
    
    with open(fname, 'rb') as fl:
        text = fl.read()
        text = u'' + text.decode('utf-8', errors='ignore')
    
    return text
text = load_text()
tokens = word_tokenize(text)
spellchecker.set_tokens(tokens)
errors = set()

for err in spellchecker:
    errors.add(err.word)
    
errors = list(errors)
doc = tsvd.transform(tfidf.transform(errors))
dict_words_vecs.shape, doc.shape
sims = cosine_similarity(doc, dict_words_vecs)
gg = list(zip([words[i] for i in sims.argmax(axis=1)], sims[:,sims.argmax(axis=1)].max(axis=1)))
errors[60]
list(enumerate(gg[:100]))
words_data = pd.read_json('./data/words_dictionary.json', typ='ser')
words_data['philippines']
"Ss s".isalpha()
w[:1000]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09515324305060585, 'fraction_numerical': 0.011047754811119031, 'mean_word_length': 3.3783151326053042, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '256465', 'license': 'MIT', 'n_tokens_mistral': 1808, 'n_tokens_neox': 1748, 'n_words': 509, 'path': 'notebooks/archive/SCRIPTS/spelling/VectorizedSpellChecker.ipynb', 'repo_name': 'worldbank/wb-nlp-apps', 'size': '5612'}