<jupyter_start><jupyter_text>--- Load in and visualize the data<jupyter_code>import numpy as np

# read data from text files
with open('data/reviews.txt', 'r') as f:
    reviews = f.read()
with open('data/labels.txt', 'r') as f:
    labels = f.read()
from string import punctuation

# get rid of punctuation
reviews = reviews.lower() # lowercase, standardize
all_text = ''.join([c for c in reviews if c not in punctuation])

# split by new lines and spaces
reviews_split = all_text.split('\n')
all_text = ' '.join(reviews_split)

# create a list of words
words = all_text.split()
# feel free to use this import 
from collections import Counter

## Build a dictionary that maps words to integers
counts = Counter(words)
vocab = sorted(counts, key=counts.get, reverse=True)
vocab_to_int = {word: ii for ii, word in enumerate(vocab, 1)}

## use the dict to tokenize each review in reviews_split
## store the tokenized reviews in reviews_ints
reviews_ints = []
for review in reviews_split:
    reviews_ints.append([vocab_to_int[word] for word in review.split()])
# 1=positive, 0=negative label conversion
labels_split = labels.split('\n')
encoded_labels = np.array([1 if label == 'positive' else 0 for label in labels_split])
# outlier review stats
review_lens = Counter([len(x) for x in reviews_ints])
print("Zero-length reviews: {}".format(review_lens[0]))
print("Maximum review length: {}".format(max(review_lens)))
print('Number of reviews before removing outliers: ', len(reviews_ints))

## remove any reviews/labels with zero length from the reviews_ints list.

# get indices of any reviews with length 0
non_zero_idx = [ii for ii, review in enumerate(reviews_ints) if len(review) != 0]

# remove 0-length reviews and their labels
reviews_ints = [reviews_ints[ii] for ii in non_zero_idx]
encoded_labels = np.array([encoded_labels[ii] for ii in non_zero_idx])

print('Number of reviews after removing outliers: ', len(reviews_ints))
from keras.preprocessing.sequence import pad_sequences
# Test your implementation!

seq_length = 200

features = pad_sequences(reviews_ints, maxlen=seq_length,  padding='pre', value=0.0)

## test statements - do not change - ##
assert len(features)==len(reviews_ints), "Your features should have as many rows as reviews."
assert len(features[0])==seq_length, "Each feature row should contain seq_length values."

# print first 10 values of the first 30 batches 
#print(features[:30,:10])

## split data into training, validation, and test data (features and labels, x and y)

from sklearn.model_selection import train_test_split

split_frac = 0.8

## split data into training, validation, and test data (features and labels, x and y)
train_x, test_x, train_y, test_y = train_test_split(features, encoded_labels, test_size= 1 - split_frac, random_state=1)
## print out the shapes of your resultant feature data
print(train_x.shape)
print(test_x.shape)
train_x.shape
from keras.layers import LSTM, Input, Dense, Dropout, Embedding
from keras.models import Sequential
from keras.layers import Activation
from keras.optimizers import Adam
from keras.losses import binary_crossentropy

# TODO: Build the model architecture
embed_dim = 128
lstm_out = 196
max_features = len(vocab_to_int)+1
model = Sequential()
model.add(Embedding(max_features, embed_dim, input_length = train_x.shape[1]))
model.add(Dropout(0.4))
model.add(LSTM(lstm_out, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1,activation='sigmoid'))
model.compile(loss = 'binary_crossentropy', optimizer='adam',metrics = ['accuracy'])
print(model.summary())

model.fit(train_x, train_y, batch_size=32, epochs=6, validation_split=0.2)
scores = model.evaluate(test_x, test_y, verbose=0)
print('Test accuracy:', scores[1])
test_str = 'What a worst movie I ever saw'
from string import punctuation

def tokenize_review(test_review):
    test_review = test_review.lower() # lowercase
    # get rid of punctuation
    test_text = ''.join([c for c in test_review if c not in punctuation])

    # splitting by spaces
    test_words = test_text.split()

    # tokens
    test_ints = []
    test_ints.append([vocab_to_int[word] for word in test_words])

    return test_ints

test_ints = tokenize_review(test_str)
#padding the tweet to have exactly the same shape as `embedding_2` input
test_ints = pad_sequences(test_ints , maxlen=200, dtype='int32', value=0)
sentiment = model.predict(test_ints)[0]>0.5
if sentiment:
    print("Positive")
else:
    print("Negative")<jupyter_output>Negative
{'fraction_non_alphanumeric': 0.10699128880947063, 'fraction_numerical': 0.013178467723922269, 'mean_word_length': 5.556368960468522, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '313307', 'license': 'MIT', 'n_tokens_mistral': 1505, 'n_tokens_neox': 1421, 'n_words': 460, 'path': 'sentiment-rnn/Sentiment_RNN_In_Keras.ipynb', 'repo_name': 'andyli1688/deep-learning-pytorch', 'size': '4477'}