<jupyter_start><jupyter_text>Deep Learning Arch 1 Embedding->CNN->LSTM->Dense<jupyter_code>from keras.layers import Embedding, Dense, Flatten, LSTM, Conv1D, MaxPooling1D, Dropout, Activation
from keras.models import Sequential
from keras.preprocessing.text import one_hot
from keras.utils import to_categorical
from keras.preprocessing.sequence import pad_sequences
import pandas as pd
import numpy as np
import os
import nltk<jupyter_output>Using TensorFlow backend.
/Users/erickmaziero/virtualenvs/CodeSmells_env/lib/python3.7/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.6 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.7
  return f(*args, **kwds)<jupyter_text>data load and preparation<jupyter_code>dataset_path = '../data/aclImdb'
train_positive_files = ['train/pos/'+f for f in os.listdir(dataset_path+'/train/pos') \
                        if os.path.isfile(os.path.join(dataset_path+'/train/pos', f))]
train_negative_files = ['train/neg/'+f for f in os.listdir(dataset_path+'/train/neg') \
                        if os.path.isfile(os.path.join(dataset_path+'/train/neg', f))]
test_positive_files = ['test/pos/'+f for f in os.listdir(dataset_path+'/test/pos') \
                       if os.path.isfile(os.path.join(dataset_path+'/test/pos', f))]
test_negative_files = ['test/neg/'+f for f in os.listdir(dataset_path+'/test/neg') \
                       if os.path.isfile(os.path.join(dataset_path+'/test/neg', f))]
all_files = list(set().union(train_positive_files,train_negative_files, test_positive_files, test_negative_files))

dataset = {'trainset':[], 
           'polarity':[], 
           'review':[]}

for file in all_files:
    polarity = file.split('.')[0].split('_')[1]
    with open(os.path.join(dataset_path, file), 'r') as text_file:
        dataset['trainset'].append(file.split('/')[0])
        dataset['polarity'].append(polarity)
        dataset['review'].append(text_file.readlines()[0])
dataframe = pd.DataFrame(data=dataset)

dataframe.head()<jupyter_output><empty_output><jupyter_text>preparation<jupyter_code>stemmer = nltk.stem.porter.PorterStemmer()
stopwords = nltk.corpus.stopwords.words('english')
stopwords = [word for word in stopwords if word not in ['very', 'no', 'nor', 'not', 'few', 'more',
                                                        'most', 'just', 'doesn',  'don', "don't", 'should', "should've", 'now', 'd', 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', "aren't", 'couldn', "couldn't", 'didn', "didn't", "doesn't", 'hadn', "hadn't", 'hasn', "hasn't", 'haven', "haven't", 'isn', "isn't", 'ma', 'mightn', "mightn't", 'mustn', "mustn't", 'needn', "needn't", 'shan', "shan't", 'shouldn', "shouldn't", 'wasn', "wasn't", 'weren', "weren't", 'won', "won't", 'wouldn', "wouldn't"]]
print(stopwords)
def preprocess_reviews(review):
    tokens = nltk.tokenize.word_tokenize(review.lower())
    stemmed_tokens = [stemmer.stem(token) for token in tokens if token not in stopwords]
    return " ".join(stemmed_tokens)

dataframe['clean_review'] = dataframe['review'].apply(preprocess_reviews)
dataframe.head()
dataframe.to_pickle('../data/dataframe_processed_reviews.pkl')
train_reviews = dataframe[dataframe['trainset'] == 'train']['clean_review'].values.tolist()
train_classes = dataframe[dataframe['trainset'] == 'train']['polarity'].values.tolist()
labels_train = np.array(train_classes)
test_reviews = dataframe[dataframe['trainset'] == 'test']['clean_review'].values.tolist()
test_classes = dataframe[dataframe['trainset'] == 'test']['polarity'].values.tolist()
labels_test = np.array(test_classes)

labels_categorical_train = to_categorical(labels_train, num_classes=11)
labels_categorical_test = to_categorical(labels_test, num_classes=11)
# find the vocab size
vocab = {}
max_length = 0
for review in train_reviews:
    tokens = review.split()
    
    if len(tokens) > max_length:
        max_length = len(tokens)
    
    for token in tokens:
        if token in vocab:
            vocab[token] += 1
        else:
            vocab[token] = 0

print(len(vocab))
print(max_length)
vocab_size = len(vocab) + 100
encoded_docs = [one_hot(d, vocab_size) for d in train_reviews]
print(encoded_docs)
padded_docs = pad_sequences(encoded_docs, maxlen=max_length, padding='post')
print(padded_docs)
vocab_size = len(vocab) + 100
encoded_docs_test = [one_hot(d, vocab_size) for d in test_reviews]
print(encoded_docs_test)
padded_docs_test = pad_sequences(encoded_docs_test, maxlen=max_length, padding='post')
print(padded_docs_test)
model1 = Sequential()
model1.add(Embedding(vocab_size, 128, input_length=max_length))
model1.add(Flatten())
model1.add(Dense(32, activation='relu'))
model1.add(Dense(11, activation='softmax'))
# compile the model
model1.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])

# summarize the model
print('################# Model 1 ###############')
print(model1.summary())
model1.fit(padded_docs, labels_categorical_train, epochs=5, verbose=1, validation_split=0.2)
# evaluate the model
loss, accuracy = model1.evaluate(padded_docs_test, labels_categorical_test, verbose=0)
print('Accuracy: %f' % (accuracy*100))
model2 = Sequential()
model2.add(Embedding(vocab_size, 128, input_length=max_length))
model2.add(Conv1D(64, 5, activation='relu'))
model2.add(MaxPooling1D(pool_size=4))
model2.add(LSTM(100))
model2.add(Dense(11, activation='softmax'))
# compile the model
model2.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
# summarize the model
print('################# Model 2 ###############')
print(model2.summary())
model2.fit(padded_docs, labels_categorical_train, epochs=5, verbose=1, validation_split=0.2)
# evaluate the model
loss, accuracy = model2.evaluate(padded_docs_test, labels_categorical_test, verbose=0)
print('Accuracy: %f' % (accuracy*100))
model1.save()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16683587140439932, 'fraction_numerical': 0.014890016920473773, 'mean_word_length': 5.641573033707865, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '131651', 'license': 'MIT', 'n_tokens_mistral': 2144, 'n_tokens_neox': 2044, 'n_words': 387, 'path': 'notebooks/DLArch1.ipynb', 'repo_name': 'egmaziero/CodeSmells', 'size': '5910'}