<jupyter_start><jupyter_text>Preprocessing English Wikipedia corpusThere are a number of corrections that will improve the quality of our word vector representations. We'll iterate on the preprocessing step to create a better word vector.Common preprocessing steps are:* dropping numbers* dropping punctuation* coercing to lower case* stemming* lemmatizationHowever, some of the operations can be undesirably lossy, e.g.: * dropping the hypen can remove certain word combinations     * the word "email" was once written "e-mail" * Coercing to lower case squashes the dimensionality of entities (capitalized nouns and Proper names)     * "Edward Said" becomes "edward said"<jupyter_code>import re

from cltk.prosody.latin.string_utils import punctuation_for_spaces_dict
from nltk.corpus import stopwords
from nltk.corpus import words as unix_dictionary
replacer = punctuation_for_spaces_dict()
print(f"Hyphen has same ascii and unicode value: {ord('-')}")
del replacer[45] 
print("Madam, I'm a Adam!".translate(replacer))
print("The word email, used to be e-mail.".translate(replacer))
print("But hyphen assimilation isn't always possible based on sense: beauty-obsessed".translate(replacer))
has_numeric = re.compile('.*[0-9]+.*')
stop_words = set(stopwords.words('english'))
lower_words = set([tmp for tmp in unix_dictionary.words() if tmp.lower() == tmp]) 
with open('wikimedia.en.processed.cor', 'wt') as writer:
    with open('wikimedia.en.cor', 'rt') as reader:
        for line in reader:            
            line = line.strip()
            words = line.translate(replacer).split()
            # Check the first word of each sentence, if it typically occurs in lower case, coerce to lower
            if words and words[0][0].upper() == words[0][0]:
                if words[0] in lower_words:
                    words[0] = words[0].lower()
            cleaned_words = [tmp for tmp in words
                                 if not has_numeric.match(tmp)                                   
                                 and len(tmp) > 1 
                                 and tmp not in stop_words ]
            writer.write(' '.join(cleaned_words))
            writer.write('\n')<jupyter_output><empty_output><jupyter_text>Now we're ready to create a quality word vector from the corpus  Appendix: notebook testing of the cleaning functions<jupyter_code># Why has_numeric
'salt-making'.isalpha()
## alpha and non-alpha match are bit problematic
# if truly needed we should scan the corpus and pull out a set of all the characters we want.
non_alpha = re.compile(r'.*[^a-zA-Z\-]+')
non_alpha.match('salt-Making')
non_alpha.match('abc120')
non_alpha.match('120abc')
has_numeric.match('abc120')
has_numeric.match('120abc')
has_numeric.match('1')
has_numeric.match('13')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10110359558561766, 'fraction_numerical': 0.009611961552153792, 'mean_word_length': 3.404388714733542, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '100398', 'license': 'Apache-2.0', 'n_tokens_mistral': 835, 'n_tokens_neox': 790, 'n_words': 270, 'path': 'wikipedia_corpus_processing/clean_eng_wiki_corpus.ipynb', 'repo_name': 'todd-cook/ML-You-Can-Use', 'size': '2809'}