<jupyter_start><jupyter_text>1. Load Data And Write Texts To Individual .txt Files<jupyter_code># # Read data from files 
# train = pd.read_csv("data/train.csv", header=0)
# test = pd.read_csv( "data/test.csv", header=0)
# print("Read %d labeled train reviews and  %d unlabelled test reviews" % (len(train),len(test)))

# train_comments = train['comment_text'].fillna("_na_").tolist()
# test_comments = test['comment_text'].fillna("_na_").tolist()
# all_comments = train['comment_text'].fillna("_na_").tolist() + test['comment_text'].fillna("_na_").tolist() 

# def comments_to_txtfile(comment_list,output_file):
#     with open(output_file, "w+") as output:
#         for comment in comment_list:
#             comment = " ".join(comment.split()).replace('"',"").replace('|',"").replace('{',"").replace('}',"")
#             comment = re.sub("[^a-zA-Z0-9]"," ",str(comment)) #Remove unneeded whitespace and punctuation and move to lowercase
#             output.write("%s\n" % str(comment))
#     return print("%s Written" % output_file)

# comments_to_txtfile(train_comments,'data/raw_comments/train_comments.csv')
# comments_to_txtfile(test_comments,'data/raw_comments/test_comments.csv')
# comments_to_txtfile(all_comments,'data/raw_comments/all_comments.csv')
def nlp_utility(input_file, 
                embeddings_file ='',
                fasttext_file = '',
                remove_stops = True,
                lemmatize = True,
                n_grams = 0, 
                fasttext_spellcheck = True,
                filter_embeddings_file = False
                ):
    
    import os,shutil,smart_open,gensim,sys,time,re
    from gensim.models import Word2Vec
    from sys import platform
    from nltk.corpus import stopwords
    from nltk.stem import WordNetLemmatizer
    from nltk.corpus import wordnet
    from nltk import pos_tag
    from gensim.models import FastText
    stops = set(stopwords.words("english"))
    
    unfiltered_glove_path = 'w2v_models/unfiltered_models/' + embeddings_file
    filtered_glove_path =  'w2v_models/filtered_models/' + 'remove_stops=' + str(remove_stops) + '.lemmatize=' + str(lemmatize) + '.spellcheck=' + str(fasttext_spellcheck) + '.' + embeddings_file
    gensim_filtered_glove_path = 'w2v_models/gensim_filtered_models/' + 'remove_stops=' + str(remove_stops) + '.lemmatize=' + str(lemmatize) + '.spellcheck=' + str(fasttext_spellcheck) + '.' + embeddings_file
    output_file = 'data/tokenized_comments/' + 'remove_stops=' + str(remove_stops) + '.lemmatize=' + str(lemmatize) + '.spellcheck=' + str(fasttext_spellcheck) + '.' + embeddings_file + '.' + input_file.split('/')[2]
    print('Result File = %s' % output_file)
    #Create a generator function to run through the input file one line at a time:
    class FileToComments(object):    
        def __init__(self, filename):
            self.filename = filename
        def __iter__(self):    
            for line in open(self.filename, 'r'):
                yield line   
    #Function used in Step 2 to match treebank tags to pos tags 
    def get_wordnet_pos(treebank_tag):
        if treebank_tag.startswith('J'):
            return wordnet.ADJ
        elif treebank_tag.startswith('V'):
            return wordnet.VERB
        elif treebank_tag.startswith('N'):
            return wordnet.NOUN
        elif treebank_tag.startswith('R'):
            return wordnet.ADV
        else:
            return ''
    def filter_vectors_gensim(unfiltered_glove_file,comment_file):
        unfiltered_glove_path = 'w2v_models/unfiltered_models/' + unfiltered_glove_file
        filtered_glove_path =  'w2v_models/filtered_models/' + 'remove_stops=' + str(remove_stops) + '.lemmatize=' + str(lemmatize) + '.spellcheck=' + str(fasttext_spellcheck) + '.' + unfiltered_glove_file
        gensim_filtered_glove_path = 'w2v_models/gensim_filtered_models/' + 'remove_stops=' + str(remove_stops) + '.lemmatize=' + str(lemmatize) + '.spellcheck=' + str(fasttext_spellcheck) + '.' + unfiltered_glove_file
        comment_streamer = FileToComments(comment_file)

        #Create a Vocabulary From Our Existing Words
        from collections import defaultdict
        import codecs, gensim
        vocab = defaultdict(int)
        for sentence in comment_streamer:
            sentence = sentence.split()
            for word in sentence:
                vocab[word] += 1
        vocab = set(vocab)
        print('Vocabulary set, starting to filter word embeddings')
        nread = 0
        nwrote = 0
        with codecs.open(unfiltered_glove_path, encoding='utf-8') as f:
            with codecs.open(filtered_glove_path, 'w', encoding='utf-8') as out:
                for line in f:
                    nread += 1
                    line = line.strip()
                    if not line: continue
                    if line.split(u' ', 1)[0] in vocab:
                        out.write(line + '\n')
                        nwrote += 1
                    sys.stdout.write("Read: %d lines. Wrote %d To Word Embedding File  \r" % (nread,nwrote))
                    sys.stdout.flush()

        sys.stdout.write("\r\n")
        print('read %s lines, wrote %s' % (nread, nwrote))
        print('Prepending Gensim W2V Line Count and Embedding Size')
        gensim.scripts.glove2word2vec.glove2word2vec(filtered_glove_path,gensim_filtered_glove_path)
        return gensim_filtered_glove_path

    #Step 1 - Split Words And Make Lowercase
    comment_streamer = FileToComments(input_file)
    counter = 0
    punctuation = re.compile(r'[-.?!,":;()|0-9]') #[punctuation and numbers to remove]
    with open('intermediate_comments_preproc.txt', "w+") as output:
        for comment in comment_streamer:
            comment = comment.lower().split()
            comment = [punctuation.sub("", word) for word in comment]    
            #Step 1 - Remove Stopwords
            if remove_stops:
                comment = [word for word in comment if not word in stops]
            
            #Step 2 - Lemmatize
            if lemmatize:
                tagged_words = pos_tag(comment)
                tokenized_words = []
                for word in tagged_words:
                    thisTag = get_wordnet_pos(word[1])
                    if thisTag == '':
                        tokenized_words.append(word[0])
                    else:
                        thisLemma = WordNetLemmatizer().lemmatize(word[0],thisTag)
                        tokenized_words.append(thisLemma)
                output.write("%s\n" % ' '.join(tokenized_words))
            else:
                output.write("%s\n" % ' '.join(comment))

            counter += 1
            sys.stdout.write("Cleaned %d comments from %s \r" % (counter,input_file))
            sys.stdout.flush()
        sys.stdout.write("\r\n")

    # Step 4 - Filter Word Embedding File (For Speed In Training)
    # We now have our commenst rewritten in a clean format with stop words removed and lemmatization in place
    # Now we need to take our chosen vector_model and sort it for words found in our sentences
    if filter_embeddings_file:
        filtered_vectors_path = filter_vectors_gensim(embeddings_file,comment_file = 'intermediate_comments_preproc.txt')
        print('Embedding file filtered and cleaned for corpora. Location : %s' % gensim_filtered_glove_path)
        return
    else:
        filtered_vectors_path = gensim_filtered_glove_path
    
    #Step 5- Use Fasttext Model For Spelling Correction
    #Load Glove Model
    glove_word_vectors = gensim.models.KeyedVectors.load_word2vec_format(filtered_vectors_path)
    print("Filtered Gensim Model Loaded")
    #Load Fasttext Model
    fasttext_vectors = FastText.load_fasttext_format(fasttext_file)
    print("Fasttext Model Loaded")
    
    intermediate_token_streamer = FileToComments('intermediate_comments_preproc.txt')
    model_vocabulary = set(glove_word_vectors.wv.vocab)
    word_subs = 0
    counter = 0
    print("Substituting mispelled words using Fasttext Model")                                      
    with open(output_file, "w+") as output:
        for wordlist in intermediate_token_streamer:
            wordlist = wordlist.split()
            output_words = []
            for word in wordlist:
                if word in model_vocabulary:
                    output_words.append(word)
                else:
                    try:
                        for pairing in fasttext_vectors.wv.most_similar(word,topn=20):
                            if pairing[0] in model_vocabulary:
                                output_words.append(pairing[0])
                                word_subs += 1
                                break
                    except:
                        next
            output.write("%s\n" % ' '.join(output_words))
            counter += 1
            sys.stdout.write("Substituted %d Mispelled Words from %d comments\r" % (word_subs,counter))
            sys.stdout.flush()
    print("Substituted %s Words. Printed to %s" % (word_subs,output_file))
    
    return
print('Available Models:')
os.listdir('w2v_models/unfiltered_models/')
# Set Params For Preprocessing
remove_stops = False
lemmatize = False
embeddings_file = 'glove.twitter.27B.200d.txt'
fasttext_file = 'fasttext-training/word_vector_models/cbow_fasttext_model.bin'
fasttext_spellcheck = True
#Filter Vectors
nlp_utility(input_file = 'data/raw_comments/all_comments.csv',
            embeddings_file =embeddings_file,
            fasttext_file = fasttext_file,
            remove_stops = remove_stops,
            lemmatize = lemmatize,
            fasttext_spellcheck = fasttext_spellcheck,
            filter_embeddings_file = True
           )
nlp_utility(input_file = 'data/raw_comments/train_comments.csv',
            embeddings_file = embeddings_file,
            fasttext_file = fasttext_file,
            remove_stops = remove_stops,
            lemmatize = lemmatize,
            fasttext_spellcheck = fasttext_spellcheck,
           )
nlp_utility(input_file = 'data/raw_comments/test_comments.csv',
            embeddings_file = embeddings_file,
            fasttext_file = fasttext_file,
            remove_stops = remove_stops,
            lemmatize = lemmatize,
            fasttext_spellcheck = fasttext_spellcheck,
           )<jupyter_output>Result File = data/tokenized_comments/remove_stops=False.lemmatize=False.spellcheck=True.glove.twitter.27B.200d.txt.test_comments.csv
Cleaned 226998 comments from data/raw_comments/test_comments.csv 
Filtered Gensim Model Loaded
Fasttext Model Loaded
Substituting mispelled words using Fasttext Model
Substituted 343384 Words. Printed to data/tokenized_comments/remove_stops=False.lemmatize=False.spellcheck=True.glove.twitter.27B.200d.txt.test_comments.csv
{'fraction_non_alphanumeric': 0.09970265749860621, 'fraction_numerical': 0.007247723471473704, 'mean_word_length': 2.516171185886965, 'pattern_counts': {'":': 1, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '424087', 'license': 'MIT', 'n_tokens_mistral': 3369, 'n_tokens_neox': 3214, 'n_words': 681, 'path': 'advanced-nlp-prepoc.ipynb', 'repo_name': 'jmoore24/kaggle-toxic-comments', 'size': '10762'}