<jupyter_start><jupyter_text>Preprocess **Text Cleanup**<jupyter_code>def text_cleanup(list_of_texts):
    """Perform text cleanup, reduce to a-z and space."""
    def cleanup(text):
        res = text.lower()
        res = regex.sub(' ', res)
        return res.split()
    
    result_cleaned = []
    regex = re.compile('[^a-z\' ]+')    # removes anything that is not a-z or space
    for text in list_of_texts:
        result_cleaned.append(cleanup(text))
    return result_cleaned                     # doubly nested list of words
train_reviews = text_cleanup(train_reviews_raw)
test_reviews = text_cleanup(test_reviews_raw)
print(train_reviews[0])
total_words, unique_words, words_counter = count_words(train_reviews)
print('Total words: ', total_words)
print('Unique words: ', unique_words)<jupyter_output>Total words:  5911692
Unique words:  85680<jupyter_text>**Remove Stopwords**<jupyter_code>train_reviews_no_stop = train_reviews
test_reviews_no_stop = test_reviews
# display(words_counter.most_common()[:10])
# import nltk
# nltk.download('stopwords')
# en_stopwords = nltk.corpus.stopwords.words('english')
# stopwords = {sw for sw in en_stopwords}
# print(stopwords)
stopwords = {'down', 'then', 'of', 'but', 'only', 'yours', 'himself', 'again',
             'very', 'or', 'once', 'until', 'have', "doesn't", 'what', 'during',
             "that'll", 'some', 'was', 'be', 'he', "should've", 'between',
             "shouldn't", 'further', 'no', 'yourself', 'm', 've', "you'll",
             'ain', 't', 'our', 'his', 'o', 'wouldn', 'below', 'any', 'under',
             'you', 'isn', 'theirs', 'why', 'that', 'mightn', 'ourselves', 'on',
             'haven', 'while', 'to', 'than', 'your', 'she', 'is', 'just',
             "mightn't", 'with', "you've", 'mustn', 'needn', 'same', 'me',
             'such', 'myself', 'there', 'own', 'this', 're', 'ma', 'from',
             'did', 'couldn', 'hasn', 'for', 'won', "won't", "mustn't", 'her',
             'can', 'doesn', "wouldn't", 'when', "you're", 'who', 'which', 'll',
             'itself', 'against', 'out', 'up', "it's", 'a', 'here', 'being',
             'they', 'as', 'didn', 'weren', 'aren', 'herself', 'the', 'if',
             "didn't", 'should', 'doing', 'other', 'has', 'so', "you'd",
             'above', 'do', 'before', 'at', 'had', 'each', "aren't", 'their',
             'now', 'an', 'through', 'how', 'those', 'nor', "hasn't", 'over',
             'by', 'into', 'themselves', 'most', 'shan', 'been', "she's",
             "haven't", "isn't", "wasn't", 'where', 'about', 'in', "hadn't",
             'because', 'too', 'whom', 'ours', 'him', 'yourselves', 'after',
             'and', 'were', 'both', 'will', 'it', 'my', 'few', 'having', 'them',
             'hadn', 'shouldn', 'does', 's', "couldn't", 'y', 'all', 'don',
             'off', 'more', 'am', 'd', 'hers', 'its', 'are', "shan't",
             "weren't", 'we', "needn't", 'i', 'these', "don't", 'wasn', 'not'}
stopwords.add('br')  # <br /> tag in a lot of reviews
# def remove_stopwords(list_of_examples, stopwords):
#     result_no_stop = []
#     for list_of_words in list_of_examples:
#         result_no_stop.append( [w for w in list_of_words if w not in stopwords])
#     return result_no_stop
# train_reviews_no_stop = remove_stopwords(train_reviews, stopwords)
# test_reviews_no_stop = remove_stopwords(test_reviews, stopwords)
# print(train_reviews_no_stop[0])
# total_words, unique_words, words_counter = count_words(train_reviews_no_stop)
# print('Total words: ', total_words)
# print('Unique words: ', unique_words)<jupyter_output><empty_output><jupyter_text>**Reduce Vocabulary**<jupyter_code>display(words_counter.most_common()[-10:])
def get_most_common_words(list_of_examples, num_words):
    words_ctr = collections.Counter()
    for example in list_of_examples:
        words_ctr.update(example)
    
    keep_words = {w for w, n in words_ctr.most_common()[:num_words]}
    return keep_words
allowed_words = get_most_common_words(train_reviews_no_stop, 9998)
print([w for w in allowed_words][:20])
def reduce_vocabulary(list_of_examples, allowed_words, unk_tok='<UNK>'):
    result_reduced = []
    for example in list_of_examples:
        result_reduced.append( [w if w in allowed_words else unk_tok for w in example] )
    return result_reduced
train_reviews_reduced = reduce_vocabulary(train_reviews_no_stop, allowed_words)
test_reviews_reduced = reduce_vocabulary(test_reviews_no_stop, allowed_words)
print(train_reviews_reduced[0])
total_words, unique_words, words_counter = count_words(train_reviews_reduced)
print('Total words: ', total_words)
print('Unique words: ', unique_words)<jupyter_output>Total words:  5911692
Unique words:  9999<jupyter_text>**Create Dictionaries**<jupyter_code>i2w = {i : w for i, (w, c) in enumerate(words_counter.most_common(), 1)}
w2i = {w : i for i, w in i2w.items()}
i2w[0] = '<PAD>'                       # use zero index for padding
w2i[i2w[0]] = 0
print('Number of words in dictionaries:', len(i2w))
for i in range(10):
    word = i2w[i]
    print(i, ':', word, ':', w2i[word])
print(sorted(list(i2w.values()))[:100])<jupyter_output>["'", "'a", "'cause", "'em", "'i", "'n'", "'s", "'the", '<PAD>', '<UNK>', 'a', 'aaron', 'abandon', 'abandoned', 'abc', 'abducted', 'abilities', 'ability', 'able', 'aboard', 'abominable', 'abomination', 'abortion', 'abound', 'about', 'above', 'abraham', 'abroad', 'abrupt', 'abruptly', 'absence', 'absent', 'absolute', 'absolutely', 'absorbed', 'absorbing', 'abstract', 'absurd', 'absurdity', 'abu', 'abundance', 'abuse', 'abused', 'abusive', 'abysmal', 'academic', 'academy', 'accent', 'accents', 'accept', 'acceptable', 'acceptance', 'accepted', 'accepting', 'accepts', 'access', 'accessible', 'accident', 'accidental', 'accidentally', 'acclaim', 'acclaimed', 'accompanied', 'accompany', 'accompanying', 'accomplish', 'accomplished', 'accomplishment', 'according', 'account', 'accounts', 'accuracy', 'accurate', 'accurately', 'accused', 'ace', 'achieve', 'achieved', 'achievement', 'achievements', 'achieves', 'achieving', 'acid', 'acknowledge', 'acknowledged', 'acquired', 'across', 'act', 'acted',[...]<jupyter_text>**Tokenize**<jupyter_code>def tokenize(list_of_examples, word2idx):
    result_tokenized = []
    for list_of_words in list_of_examples:
        result_tokenized.append( [word2idx[w] for w in list_of_words] )
    return result_tokenized
train_reviews_tok = tokenize(train_reviews_reduced, w2i)
test_reviews_tok = tokenize(test_reviews_reduced, w2i)
print(train_reviews_tok[0])
def encode_multihot(arr_of_tokens, encode_size):
    result_encoded = np.zeros(shape=(len(arr_of_tokens), encode_size))
    for i, tokens in enumerate(arr_of_tokens):
        result_encoded[i, tokens] = 1
    return result_encoded  # numpy array
train_reviews_enc = encode_multihot(train_reviews_tok, encode_size=len(i2w))
test_reviews_enc = encode_multihot(test_reviews_tok, encode_size=len(i2w))
print(train_reviews_enc[0, :100])<jupyter_output>[0. 1. 1. 1. 1. 1. 1. 1. 0. 1. 1. 1. 0. 1. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0.
 0. 0. 0. 0. 0. 1. 1. 1. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 1.
 0. 1. 0. 0. 1. 0. 0. 0. 1. 0. 0. 1. 0. 1. 0. 0. 0. 1. 1. 0. 1. 0. 1. 0.
 1. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 1. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
 0. 0. 0. 0.]<jupyter_text>**Convert Labels**<jupyter_code>train_labels = np.array(train_labels_raw)
test_labels = np.array(test_labels_raw)
print(train_labels)<jupyter_output>[1 1 1 ... 0 0 0]<jupyter_text>Sklearn Model<jupyter_code>import sklearn.metrics
from sklearn import linear_model
model = linear_model.LogisticRegression(C=0.05, solver='liblinear')
model.fit(train_reviews_enc, train_labels)
predictions = model.predict(test_reviews_enc)
sklearn.metrics.accuracy_score(test_labels, predictions)
raise # sentinel, don't auto-run beyond this point<jupyter_output><empty_output><jupyter_text>**10,000 words vocab** Reference model<jupyter_code>sklearn.metrics.accuracy_score(test_labels, predictions)<jupyter_output><empty_output><jupyter_text>Stopwords kept in<jupyter_code>sklearn.metrics.accuracy_score(test_labels, predictions)<jupyter_output><empty_output><jupyter_text>This is best modelConclusions are as follows:* use 10k vocabulary size* keep apostrophe in words, e.g. "don't", "desn't"* don't remove stopwordsBot Stopwords kept in, and apostrophe kept in "'"<jupyter_code>sklearn.metrics.accuracy_score(test_labels, predictions)<jupyter_output><empty_output><jupyter_text>Stopwords kept in, no cleanup at all (just split and tolower)<jupyter_code>sklearn.metrics.accuracy_score(test_labels, predictions)<jupyter_output><empty_output><jupyter_text>**60,000 words vocab**<jupyter_code>sklearn.metrics.accuracy_score(test_labels, predictions)<jupyter_output><empty_output><jupyter_text>**Ablation Study - 1000 words vocab** Reference Model<jupyter_code>sklearn.metrics.accuracy_score(test_labels, predictions)<jupyter_output><empty_output><jupyter_text>Cleanup - no cleanup, just tolower()<jupyter_code>sklearn.metrics.accuracy_score(test_labels, predictions)<jupyter_output><empty_output><jupyter_text>Cleanup - keep apostrophe in "'"<jupyter_code>sklearn.metrics.accuracy_score(test_labels, predictions)<jupyter_output><empty_output><jupyter_text>Stop-words not removed - 'a', 'an', 'the' are still in dataset<jupyter_code>sklearn.metrics.accuracy_score(test_labels, predictions)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17626397019691326, 'fraction_numerical': 0.0204364023416711, 'mean_word_length': 5.0894361633182115, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '187774', 'license': 'MIT', 'n_tokens_mistral': 3680, 'n_tokens_neox': 3409, 'n_words': 768, 'path': 'SKLearn/IMDB.ipynb', 'repo_name': 'marcinbogdanski/ai-sketchpad', 'size': '9395'}