<jupyter_start><jupyter_text>LSTM<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import accuracy_score
from sklearn.multiclass import OneVsRestClassifier
from sklearn.metrics import confusion_matrix
from sklearn.feature_extraction.text import TfidfVectorizer
import seaborn as sns
from sklearn.metrics import classification_report
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from keras.models import Model
from keras.layers import LSTM, Activation, Dense, Dropout, Input, Embedding
from keras.optimizers import RMSprop
from keras.preprocessing.text import Tokenizer
from keras.preprocessing import sequence
from keras.utils import to_categorical
from keras.callbacks import EarlyStopping
%matplotlib inline
# Read dataset to dataframe
df = pd.read_csv('../../data/processed/preprocessed_cupid.csv', usecols=['age', 'sex','#anwps', 'text', 'clean_text', 'isced', 'isced2'])
df = df.dropna(subset=['clean_text', 'isced'])
df.head()
df1 = df
print("unique class values in the original dataset:", df1['isced'].unique())
print("classes count table: ", pd.crosstab(index=df1['isced'], columns='count'))

# Merge class 1.0, 3.0 and 5.0
df1['isced'].mask(df1['isced'].isin([3.0, 5.0]) , 1.0, inplace=True)
# Merge class 6.0, 7.0 and 8.0
df1['isced'].mask(df1['isced'].isin([6.0, 7.0]) , 8.0, inplace=True)
print('unique class values after merging:', df1['isced'].unique())
print("count table after merging:", pd.crosstab(index=df1['isced'], columns='count'))
# classes distribution
cnt_isced = df1['isced'].value_counts()
plt.figure(figsize=(12,4))
sns.barplot(cnt_isced.index, cnt_isced.values, alpha=0.8)
plt.ylabel('Number of Occurrences', fontsize=12)
plt.xlabel('isced', fontsize=12)
plt.xticks(rotation=90)
plt.show();
df1 = df1.dropna(subset=['clean_text', 'isced'])
X = df1.clean_text
Y = df1.isced

le = LabelEncoder()
Y = le.fit_transform(Y)
Y = Y.reshape(-1,1)
X_train,X_test,Y_train,Y_test = train_test_split(X,Y,test_size=0.15)


max_words = 2000
max_len = 150
tok = Tokenizer(num_words=max_words)
tok.fit_on_texts(X_train)
sequences = tok.texts_to_sequences(X_train)
sequences_matrix = sequence.pad_sequences(sequences,maxlen=max_len)
import keras

class_weight = {0 : 0.70,
                1 : 0.30}<jupyter_output><empty_output><jupyter_text>RNN<jupyter_code>def RNN():
    inputs = Input(name='inputs', shape=[max_len])
    layer  = Embedding(max_words, 50, input_length=max_len)(inputs)
    layer  = LSTM(64)(layer)
    layer  = Dense(256,name='FC1')(layer)
    layer  = Activation('relu')(layer)
    layer  = Dropout(0.5)(layer)
    layer  = Dense(1,name='out_layer')(layer)
    layer  = Activation('sigmoid')(layer)
    model  = Model(inputs=inputs,outputs=layer)
    return model
model = RNN()
model.summary()
model.compile(loss='binary_crossentropy', optimizer=RMSprop(), metrics=['accuracy'])

model.fit(sequences_matrix, Y_train, batch_size=128, epochs=10, class_weight=class_weight,
          validation_split=0.2, callbacks=[EarlyStopping(monitor='val_loss', min_delta=0.0001)])
test_sequences = tok.texts_to_sequences(X_test)
test_sequences_matrix = sequence.pad_sequences(test_sequences,maxlen=max_len)
accr = model.evaluate(test_sequences_matrix,Y_test)
print('Test set\n  Loss: {:0.3f}\n  Accuracy: {:0.3f}'.format(accr[0],accr[1]))
prediction = model.predict(test_sequences_matrix)
# import tensorflow as tf
# cm = tf.contrib.metrics.confusion_matrix(Y_test, prediction)
prediction
y_pred = []
for i in range(len(prediction)):
    if prediction[i] > 0.5: 
        y_pred.append(1) 
    else: 
        y_pred.append(0)
cm = confusion_matrix(Y_test,y_pred)
print(cm)

print(sklearn.metrics.classification_report(Y_test, y_pred))<jupyter_output>[[ 935  694]
 [ 670 5190]]
              precision    recall  f1-score   support

           0       0.58      0.57      0.58      1629
           1       0.88      0.89      0.88      5860

    accuracy                           0.82      7489
   macro avg       0.73      0.73      0.73      7489
weighted avg       0.82      0.82      0.82      7489
{'fraction_non_alphanumeric': 0.12753222836095765, 'fraction_numerical': 0.040976058931860036, 'mean_word_length': 4.72463768115942, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '97209', 'license': 'MIT', 'n_tokens_mistral': 1646, 'n_tokens_neox': 1530, 'n_words': 320, 'path': 'docs/reports/report4_LSTM.ipynb', 'repo_name': 'ShNadi/OkCupid-project', 'size': '4344'}