<jupyter_start><jupyter_text>Clean text for analysis   Import modules<jupyter_code>from connect_to_mongo import connect_to_mongo
import nltk
import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
from nltk.corpus import stopwords
from nltk.stem import porter<jupyter_output><empty_output><jupyter_text>Connect to Mongo database<jupyter_code>db, transcipts = connect_to_mongo('x-files', 'transcripts')<jupyter_output><empty_output><jupyter_text>Format transcripts for basic text analysis  Treat each show as a separate text entry. All of Scully's lines will be one entry, all of Mulder's lines another.<jupyter_code>doc_cursor = transcipts.find()
scully_corpus = []
mulder_corpus = []
for i in doc_cursor:
    scully = i['scully_lines']
    scully_concat = (' ').join(scully)
    scully_corpus.append(scully_concat)
    mulder = i['mulder_lines']
    mulder_concat = (' ').join(mulder)
    mulder_corpus.append(mulder_concat)
len(scully_corpus)
scully_corpus[0]
scully_concat
mulder_concat = (" ").join(mulder)<jupyter_output><empty_output><jupyter_text>Process all lines as one text entry   Lowercase<jupyter_code>lower_case = scully_corpus[0].lower()
lower_case[:400]<jupyter_output><empty_output><jupyter_text>Important words<jupyter_code>important_words = {'f.b.i.': 'fbi', 'x-files': 'xfiles', 'x-file': 'xfile'}
for key in important_words.keys():
    lower_case = lower_case.replace(key, important_words[key])
lower_case<jupyter_output><empty_output><jupyter_text>Remove punctuation  Replace punctuation with a whitespace, for easier removal of stopwords later like I've (stopwords that appear as separate entries in the corpus - for I've it appears as 'i' and 've' in the stopword corpus).<jupyter_code>import re
import string
punc_removed = re.sub('[%s]' % re.escape(string.punctuation), ' ', lower_case)
# remove extra whitespaces
# regex for 2 or more whitespaces, replace with 1 whitespace
punc_removed[:100]<jupyter_output><empty_output><jupyter_text>Remove words with digits<jupyter_code>no_digits = re.sub('\w*\d\w*', '', punc_removed)
no_digits[:400]<jupyter_output><empty_output><jupyter_text>Replace 2 or more whitespaces with a single space.<jupyter_code>space_removed = " ".join(no_digits.split())
space_removed[:400]<jupyter_output><empty_output><jupyter_text>Stem & Stopwords Stopwords can be removed as part of count vectorizer, but we'll do it here separately.<jupyter_code>my_stopwords = stopwords.words('english')
stemmer = nltk.stem.porter.PorterStemmer()
stemmed_text = []
for word in space_removed.split():
    if word not in my_stopwords: 
        word_stem = stemmer.stem(word)
        stemmed_text.append(word_stem)
stemmed_text = (' ').join(stemmed_text)
stemmed_text<jupyter_output><empty_output><jupyter_text>Preprocessing pipeline<jupyter_code>import re
import string
special_words = {'f.b.i.': 'fbi', 'x-files': 'xfiles', 'x-file': 'xfile'}

# try other nltk stemmers...snowball stemmer

def clean_text(corpus, use_stemmer = True):
    """Add a description here"""
    cleaned_text = []
    for text in corpus:
        lower_case = text.lower()
        for key in special_words.keys():
            preserved_words = lower_case.replace(key, special_words[key])
        punc_removed = re.sub('[%s]' % re.escape(string.punctuation), ' ', preserved_words)
        no_digits = re.sub('\w*\d\w*', '', punc_removed)
        space_removed = " ".join(no_digits.split()) # check if this is redundant
        clean_text = ''
        keeper_words = []
        if use_stemmer:
            for word in space_removed.split():
                if word not in my_stopwords: 
                    word_stem = stemmer.stem(word) # wrap this in ifelse() for stemmer
                    keeper_words.append(word_stem)
        else:
            for word in space_removed.split():
                if word not in my_stopwords:
                    keeper_words.append(word)
        clean_text = (' ').join(keeper_words)
        cleaned_text.append(clean_text)
    return(cleaned_text)
scully_cleaned = clean_text(scully_corpus)
mulder_cleaned = clean_text(mulder_corpus)
len(scully_cleaned) == len(scully_corpus)
scully_cleaned[15]<jupyter_output><empty_output><jupyter_text>Basic Counts  Top words for Scully and Mulder over the whole show  Append all the episodes together and see what each character says most often.   Adding labels to documents for classification - testing if Scully or Mulder said it  Make a function that gets the length of the scully or mulder list and creates an array with the data labels for each record.<jupyter_code>import numpy as np

label_dict = {'Scully': 0, 'Mulder': 1}

def get_labels(list_of_docs, character):
    number_of_docs = len(list_of_docs)
    labels = np.array([label_dict[character]] * number_of_docs)
    return labels
scully_list = [scully_concat]
scully_list.append('test')
get_labels(scully_list, 'Scully')<jupyter_output><empty_output><jupyter_text>Count vectorize<jupyter_code>cv = CountVectorizer(lowercase=True, stop_words='english')
cv_data = cv.fit_transform([scully_concat])
scully_count_vect = pd.DataFrame(cv_data.toarray(), columns=cv.get_feature_names())
scully_count_vect
scully_count_vect.sort_values(by=0, axis=1, ascending=False)
cv = CountVectorizer(lowercase=True, stop_words='english')
cv_data = cv.fit_transform([mulder_concat])
mulder_count_vect = pd.DataFrame(cv_data.toarray(), columns=cv.get_feature_names())
mulder_count_vect
list(mulder_count_vect.columns)
mulder_count_vect.sort_values(by=0, axis=1, ascending=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1070721612380781, 'fraction_numerical': 0.004498830304120929, 'mean_word_length': 5.27313769751693, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '284829', 'license': 'MIT', 'n_tokens_mistral': 1924, 'n_tokens_neox': 1872, 'n_words': 417, 'path': '2_clean_text.ipynb', 'repo_name': 'cstepien/the_x_files_text_analysis', 'size': '5557'}