<jupyter_start><jupyter_text>Part 2: Advanced Techniques<jupyter_code>import nltk
import numpy as np
import pandas as pd
pd.set_option('display.max_colwidth', 100)
import string
import re
#from nltk.corpus import stopwords
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import precision_score, recall_score
from sklearn.model_selection import train_test_split
import gensim
import gensim.downloader as api

from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences

import keras.backend as K
from keras.layers import Dense, Embedding, LSTM
from keras.models import Sequential

import matplotlib.pyplot as plt
from IPython.display import IFrame
messages = pd.read_csv('data/spam.csv', encoding='latin-1')
messages.head()
messages = messages.drop(labels= ["Unnamed: 2", 
                                  "Unnamed: 3",
                                  "Unnamed: 4"],
                        axis=1)
messages.columns = ["label", "text"]
messages.head()<jupyter_output><empty_output><jupyter_text>Check missing values and class imbalances<jupyter_code>messages['label'].value_counts()
print("number of nulls in labels: {}".format(
    messages['label'].isnull().sum()))
print("number of nulls in text: {}".format(
    messages['text'].isnull().sum()))<jupyter_output>number of nulls in labels: 0
number of nulls in text: 0<jupyter_text>Remove punctuation<jupyter_code>string.punctuation
def remove_punct(text):
    text = "".join([char for char in text if char not in string.punctuation])
    return text

messages['text_clean'] = messages['text'].apply(
lambda x : remove_punct(x))

messages.head(3)<jupyter_output><empty_output><jupyter_text>Tokenize<jupyter_code>def tokenize(text_clean):
    tokens = re.split('\W+', text_clean)
    return tokens

messages['text_tokenized'] = messages['text_clean'].apply(
lambda x : tokenize(x.lower()))

messages.head(3)<jupyter_output><empty_output><jupyter_text>Remove Stopwords<jupyter_code>stopwords = nltk.corpus.stopwords.words('english')
def remove_stopwords(tokenized_text):
    text = [word for word in tokenized_text if word not in stopwords.words('english')]
    
    return text

messages['text_nostop'] = messages['text_tokenized'].apply(
    lambda x : remove_stopwords(x))

messages.head(3)<jupyter_output><empty_output><jupyter_text>Vectorizing using TF-IDF<jupyter_code># Redifine text prep function to do everything in one go:
def clean_text(text):
    text = "".join([word.lower() for word in text if word not in string.punctuation])
    tokens = re.split('\W+', text)
    text = [word for word in tokens if word not in stopwords]
    return text
tfidf_vect = TfidfVectorizer(analyzer=clean_text)
X_tfidf = tfidf_vect.fit_transform(messages['text'])
print(X_tfidf.shape)
print(tfidf_vect.get_feature_names())
# How is the output of TfidfVectorizer stored?
X_features = pd.DataFrame(X_tfidf.toarray())
X_features.head(3)<jupyter_output><empty_output><jupyter_text>Random Forest benchmark model<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(
    X_features, messages['label'], test_size=0.2)
# Create and teain the model 
rf = RandomForestClassifier()
rf_model = rf.fit(X_train, y_train)
# Make prerf_modeldelons
y_pred = rf_model.predict(X_test)
# Evaluate the model based on precision and recall scores
precision = precision_score(y_test, y_pred, pos_label='spam')
recall = recall_score(y_test, y_pred, pos_label='spam')
print("Precision: {} / Recall: {}".format(round(
precision, 3), round(recall, 3)))
print("""Conclusion: based on recall of {} the model is not aggressive enough in identifying spam""".format(round(recall, 3)))<jupyter_output>Precision: 1.0 / Recall: 0.783
Conclusion: based on recall of 0.783 the model is not aggressive enough in identifying spam<jupyter_text>Word2vec using skip-gram method*You should know a word by the company it keeps...***Some other options:**- `glove-twitter-{25/50/100/200}`- `glove-wiki-gigaword-{50/200/300}`- `word2vec-google-news-300`- `word2vec-ruscorpa-news-300`<jupyter_code>#!pip install -U gensim
#import gensim.downloader as api
wiki_embeddings = api.load('glove-wiki-gigaword-100')
# Usage example: expolore tghe word vecto for 'king'
wiki_embeddings['king']
# Find the words most similar to 'king' based on the trained word vectors
wiki_embeddings.most_similar('king')
wiki_embeddings.distances('king', 'son')
# Usage example of alternative data cleaning library
data = messages[['label', 'text']].copy() # Take a copy of orginal dataset
data['text_clean'] = data['text'].apply(
lambda x: gensim.utils.simple_preprocess(x))

data.head(3)
X_train, X_test, y_train, y_test = train_test_split(
data['text_clean'], data['label'], test_size=0.2)
# Train word2vec model
w2v_model = gensim.models.Word2Vec(X_train, 
                                   size=100, 
                                   window=5,
                                   min_count=2)

w2v_model.wv['king']
w2v_model.wv.most_similar('king')
# Here is a list of words we learned from, i.e. these words appeared at leats twice
len(w2v_model.wv.index2word)
# Generate aggregated sentence vectors based on the word vectors
# for each word in the sentence
w2v_vect = np.array([np.array([w2v_model.wv[i]
                              for i in ls
                              if i in w2v_model.wv.index2word])
                    for ls in X_test])
# Why is the length of the sentence different than the legth
# of the sentece vector?
for i, v in enumerate(w2v_vect):
    print(len(X_test.iloc[i]), len(v))
# Compute sentence vectors by averaging the word vectors
# for the words contained in the sentence
w2v_vect_avg = []
for vect in w2v_vect:
    if len(vect) != 0:
        w2v_vect_avg.append(vect.mean(axis=0))
    else:
        w2v_vect_avg.append(np.zeros(100))
# Are the lengths consitent now? Yes!
for i, v in enumerate(w2v_vect_avg):
    print(len(X_test.iloc[i]), len(v))<jupyter_output>30 100
12 100
6 100
8 100
7 100
6 100
23 100
25 100
17 100
25 100
25 100
8 100
19 100
14 100
30 100
6 100
20 100
7 100
6 100
11 100
23 100
11 100
19 100
13 100
19 100
21 100
14 100
19 100
23 100
28 100
28 100
14 100
5 100
6 100
31 100
4 100
10 100
19 100
11 100
6 100
1 100
27 100
8 100
7 100
10 100
13 100
28 100
4 100
20 100
5 100
19 100
5 100
29 100
23 100
12 100
18 100
8 100
21 100
21 100
5 100
24 100
18 100
26 100
12 100
14 100
7 100
2 100
14 100
18 100
3 100
13 100
6 100
9 100
2 100
9 100
19 100
37 100
1 100
24 100
28 100
14 100
19 100
15 100
1 100
22 100
11 100
6 100
17 100
7 100
6 100
8 100
5 100
19 100
15 100
23 100
20 100
13 100
26 100
11 100
4 100
35 100
5 100
15 100
10 100
8 100
6 100
4 100
6 100
2 100
9 100
11 100
8 100
18 100
4 100
15 100
10 100
9 100
7 100
4 100
10 100
4 100
24 100
7 100
19 100
10 100
9 100
6 100
11 100
7 100
13 100
7 100
7 100
36 100
28 100
15 100
23 100
2 100
8 100
4 100
6 100
7 100
20 100
17 100
6 100
8 100
5 100
3 100
17 100
20 100
21 100
46 100
27 100[...]<jupyter_text>Doc2vec - shallow two-layer neural network<jupyter_code># Create tagged document objects to prepare to train the model
tagged_docs = [gensim.models.doc2vec.TaggedDocument(v, [i]) 
                for i, v in enumerate(X_train)]
tagged_docs[0]
# Train a basic doc2vec model
d2v_model = gensim.models.Doc2Vec(tagged_docs, 
                                 vector_size=100,
                                 window=5,
                                 min_count=2)
d2v_model.infer_vector(['i', 'am', 'learning', 'NLP'])<jupyter_output><empty_output><jupyter_text>How to prepare document vectors for modeling<jupyter_code>d2v_model.infer_vector(['convert', 'words', 'to', 'vectors'])[:5]
d2v_model.infer_vector(['convert', 'to', 'words', 'vectors'])[:5]
# How to prepare these vectors to be used in a machine learning model/
vectors = [[d2v_model.infer_vector(words)] for words in X_test]
len(vectors)<jupyter_output><empty_output><jupyter_text>Recurrent Neural NetworkPattern matching through the connection of many very simplefunctions to create one very powerful function; **this functionhas an understanding of the data's sequential nature(using feedback loops that form a sense of memory**)<jupyter_code>IFrame("images/rnn.pdf'", width=700, height=300)
 data.head()
messages = pd.read_csv('data/spam.csv', encoding='latin-1')
messages = messages.drop(labels = ["Unnamed: 2", "Unnamed: 3", "Unnamed: 4"], axis = 1)
messages.columns = ["label", "text"]
labels = np.where(messages['label']=='spam', 1, 0)

X_train, X_test, y_train, y_test = train_test_split(messages['text'],
                                                    labels, test_size=0.2)
labels
# Initialize and fit the tokenizer
tokenizer = Tokenizer()
tokenizer.fit_on_texts(X_train)
# Use that tokenizer to transform the text messages
# in the training and test sets
X_train_seq = tokenizer.texts_to_sequences(X_train)
X_test_seq = tokenizer.texts_to_sequences(X_test)
# Numeric representation of a single message... 
# Note: variable length
X_train_seq[0]
# Need to pad the sequences so each sequence is the same length
X_train_seq_padded = pad_sequences(X_train_seq, 50) # adding 0 if not long enought
X_test_seq_padded = pad_sequences(X_test_seq, 50) # adding 0 if not long enought
type(X_train_seq_padded)
# What do these padded sequences look like?
X_train_seq_padded[88]<jupyter_output><empty_output><jupyter_text>Building the RNN model,m yeah!<jupyter_code># import keras.backend as K
# from keras.layers import Dense, Embedding, LSTM
# from keras.models import Sequential

def recall_m(y_true, y_pred):
        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
        recall = true_positives / (possible_positives + K.epsilon())
        return recall

def precision_m(y_true, y_pred):
        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
        precision = true_positives / (predicted_positives + K.epsilon())
        return precision
# Construct a simple RNN model
model = Sequential()

model.add(Embedding(len(tokenizer.index_word)+1, 32))
model.add(LSTM(32, dropout=0, recurrent_dropout=0))
model.add(Dense(32, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
model.summary()
# Compile the model
model.compile(optimizer='adam',
              loss='binary_crossentropy',
              metrics=['accuracy', precision_m, recall_m])

# model.compile(optimizer='adam', 
#               loss='binary_crossentropy',
#               metrics=['accuracy', precision_m, recall_m])
# Fit the model
history = model.fit(X_train_seq_padded, y_train, 
                    batch_size=32, epochs=10,
                    validation_data=(X_test_seq_padded, y_test))


# history = model.fit(X_train_seq_padded,
#                     y_train,
#                     batch_size=32,
#                     epochs=1,
#                     validation_data=(X_test_seq_padded, y_test))
for i in ['accuracy', 'precision_m', 'recall_m']:
    acc = history.history[i]
    val_acc = history.history['val_{}'.format(i)]
    epochs = range(1, len(acc) + 1)

    plt.figure()
    plt.plot(epochs, acc, label='Training Accuracy')
    plt.plot(epochs, val_acc, label='Validation Accuracy')
    plt.title('Results for {}'.format(i))
    plt.legend()
    plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10642201834862386, 'fraction_numerical': 0.07470511140235911, 'mean_word_length': 3.9464131374243734, 'pattern_counts': {'":': 0, '<': 46, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '483620', 'license': 'MIT', 'n_tokens_mistral': 4610, 'n_tokens_neox': 3902, 'n_words': 1158, 'path': 'nlp_advanced.ipynb', 'repo_name': 'allaccountstaken/nlp_with_nltk', 'size': '11445'}