<jupyter_start><jupyter_text>The data is loaded with the 10,000 most common words. This doesn't use the more rare words, so that we can keep the data size manageable... recommended by the official tutorial.<jupyter_code>print(train_data[0])
len(train_data[0]), len(train_data[1])
word_dic = imdb.get_word_index()

word_dic = {key: (v+3) for key,v in word_dic.items()} 
word_dic["<PAD>"] = 0
word_dic["<START>"] = 1
word_dic["<UNK>"] = 2
word_dic["<UNUSED>"] = 3

reverse_word_index = dict([(value, key) for (key, value) in word_dic.items()])

def decode_review(text):
    return ' '.join([reverse_word_index.get(i, "?") for i in text])
decode_review(train_data[0])
train_data = keras.preprocessing.sequence.pad_sequences(train_data,
                                                        value=word_index["<PAD>"],
                                                        padding='post',
                                                        maxlen=256)


test_data = keras.preprocessing.sequence.pad_sequences(test_data,
                                                       value=word_index["<PAD>"],
                                                       padding='post',
                                                       maxlen=256)
print(train_data[0])
vocab_size = 10000

model = keras.Sequential()
model.add(keras.layers.Embedding(vocab_size, 16))
model.add(keras.layers.GlobalAveragePooling1D())
model.add(keras.layers.Dense(16, activation = tf.nn.relu))
model.add(keras.layers.Dense(1, activation = tf.nn.sigmoid))

model.summary()
model.compile(optimizer=tf.train.AdamOptimizer(),
              loss='binary_crossentropy',
              metrics=['accuracy'])
x_val = train_data[:10000]
partial_x_train = train_data[10000:]

y_val = train_labels[:10000]
partial_y_train = train_labels[10000:]
history = model.fit(partial_x_train,
                    partial_y_train,
                    epochs=40,
                    batch_size=512,
                    validation_data=(x_val, y_val),
                    verbose=1)
results = model.evaluate(test_data, test_labels)

print(results)

history_dict = history.history
history_dict.keys()
import matplotlib.pyplot as plt

acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = range(1, len(acc) + 1)

plt.plot(epochs, loss, 'bo', label = 'Training loss')
plt.plot(epochs, val_loss, 'b', label = 'Validation loss')
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()

plt.figure()
acc_values = history_dict['acc']
val_acc_values = history_dict['val_acc']

plt.plot(epochs, acc, 'bo', label = 'Training acc')
plt.plot(epochs, val_acc, 'b', label = 'Validation acc')
plt.title('Training and validation accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend()

plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13365483982087495, 'fraction_numerical': 0.020668274199104376, 'mean_word_length': 2.8514588859416445, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '472528', 'license': 'MIT', 'n_tokens_mistral': 988, 'n_tokens_neox': 915, 'n_words': 178, 'path': 'Python/d024_first_tf_keras_models/.ipynb_checkpoints/d024_1_text_classification-checkpoint.ipynb', 'repo_name': 'yashaslokesh/100-Days-Of-Code', 'size': '2903'}