<jupyter_start><jupyter_text>Encoder<jupyter_code>encoder = info.features['text'].encoder
sample_string = 'Hello TensorFlow.'
encoded_string = encoder.encode(sample_string)
print ('Encoded string is {}'.format(encoded_string))<jupyter_output>Encoded string is [4025, 222, 6307, 2327, 4043, 2120, 7975]<jupyter_text>Prepare the data for training<jupyter_code>BUFFER_SIZE = 1000

train_batches = (
    train_data
    .shuffle(BUFFER_SIZE)
    .padded_batch(32, train_data.output_shapes))

test_batches = (
    test_data
    .padded_batch(32, train_data.output_shapes))
BUFFER_SIZE = 1000

train_batches = (
    train_data
    .shuffle(BUFFER_SIZE)
    .padded_batch(32, train_data.output_shapes))

test_batches = (
    test_data
    .padded_batch(32, train_data.output_shapes))<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>model = keras.Sequential([
  keras.layers.Embedding(encoder.vocab_size, 16),
  keras.layers.GlobalAveragePooling1D(),
  keras.layers.Dense(1, activation='sigmoid')])

model.summary()
model.compile(optimizer='adam',
              loss='binary_crossentropy',
              metrics=['accuracy'])<jupyter_output><empty_output><jupyter_text>Train the Model<jupyter_code>history = model.fit(train_batches,
                    epochs=10,
                    validation_data=test_batches,
                    validation_steps=30)<jupyter_output>Epoch 1/10
782/782 [==============================]949 - accuracy: 0.46 - 1s 532ms/step - loss: 0.6916 - accuracy: 0.578 - 1s 363ms/step - loss: 0.6919 - accuracy: 0.562 - 1s 278ms/step - loss: 0.6924 - accuracy: 0.546 - 1s 226ms/step - loss: 0.6923 - accuracy: 0.550 - 1s 193ms/step - loss: 0.6929 - accuracy: 0.531 - 1s 169ms/step - loss: 0.6937 - accuracy: 0.500 - 1s 150ms/step - loss: 0.6941 - accuracy: 0.484 - 1s 136ms/step - loss: 0.6936 - accuracy: 0.496 - 1s 124ms/step - loss: 0.6935 - accuracy: 0.500 - 1s 115ms/step - loss: 0.6938 - accuracy: 0.485 - 1s 107ms/step - loss: 0.6938 - accuracy: 0.487 - 1s 100ms/step - loss: 0.6934 - accuracy: 0.502 - 1s 95ms/step - loss: 0.6936 - accuracy: 0.497 - 1s 89ms/step - loss: 0.6935 - accuracy: 0.49 - 1s 85ms/step - loss: 0.6933 - accuracy: 0.50 - 1s 82ms/step - loss: 0.6934 - accuracy: 0.50 - 1s 78ms/step - loss: 0.6934 - accuracy: 0.50 - 1s 75ms/step - loss: 0.6935 - accuracy: 0.49 - 1s 72ms/step - loss: 0.6934 - accuracy: 0.49 - 1s 70ms/[...]<jupyter_text>Evaluate the Model<jupyter_code>history_dict = history.history
history_dict.keys()
import matplotlib.pyplot as plt

acc = history_dict['accuracy']
val_acc = history_dict['val_accuracy']
loss = history_dict['loss']
val_loss = history_dict['val_loss']

epochs = range(1, len(acc) + 1)

# "bo" is for "blue dot"
plt.plot(epochs, loss, 'bo', label='Training loss')
# b is for "solid blue line"
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()

plt.show()
plt.clf()   # clear figure

plt.plot(epochs, acc, 'bo', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and validation accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend(loc='lower right')

plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1526835286859963, 'fraction_numerical': 0.09407772979642195, 'mean_word_length': 3.973926380368098, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '161155', 'license': 'Apache-2.0', 'n_tokens_mistral': 1462, 'n_tokens_neox': 1207, 'n_words': 254, 'path': 'notebook/00-Test/Raphael/old/07-tensorflow.ipynb', 'repo_name': 'tarrade/proj_NLP_text_classification_with_GCP', 'size': '3242'}