<jupyter_start><jupyter_text><jupyter_code>import tensorflow_datasets as tfds
import numpy as np

from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences

from keras.models import Sequential
from tensorflow.keras.layers import Embedding

from keras.layers import Flatten
from keras.layers import Dense

import matplotlib.pyplot as plt

import io
imdb, info  = tfds.load('imdb_reviews', with_info=True, as_supervised=True)
train_data, test_data = imdb['train'], imdb['test']
train_sentences = []
test_sentences =[]

train_labels = []
test_labels=[]

for s, l in train_data:
  train_sentences.append(str(s.numpy()))
  train_labels.append(l.numpy())

for s,l in test_data:
  test_sentences.append(str(s.numpy()))
  test_labels.append(l.numpy())
train_sentences[0]
train_sentences, train_labels = np.array(train_sentences), np.array(train_labels)
test_sentences, test_labels = np.array(train_sentences), np.array(test_labels)
train_sentences.shape,  test_sentences.shape
label_sentences
# A typical sentence is 140-word length. We have two ways to represent it
# One-hot word vectors with vocabulary size equal 10000 words/ dimensions 
# Word Embeddings with 64 dimensions
# Matrix (140, 1000) x (1000, 64) = (140, 60)

vocab_size = 10000    # Maximum size of dictionary
embedding_dim = 64    # Hidden Layer dimension to describe a word
max_length = 140      # Max length of sentence

tokenizer = Tokenizer(num_words=vocal_size, oov_token="<OOV>")
tokenizer.fit_on_texts(train_sentences)
train_sequences = tokenizer.texts_to_sequences(train_sentences)
padded_train_sequences = pad_sequences(train_sequences, maxlen=max_length, 
                                       truncating='post', padding='post')
padded_train_sequences.shape
test_sequences = tokenizer.texts_to_sequences(test_sentences)
padded_test_sequences = pad_sequences(test_sequences, maxlen=max_length ,
                                      padding='post', truncating='post')
padded_test_sequences.shape
def build_model():
  model = Sequential()
  model.add(Embedding(input_dim=vocal_size, output_dim=embedding_dim, 
                      input_length=max_length))
  model.add(Flatten())
  model.add(Dense(10, activation='relu'))
  model.add(Dense(1, activation='sigmoid'))

  model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['acc'])

  return model

model = build_model()
history = model.fit(padded_train_sequences, train_labels, epochs=20, 
          validation_data=(padded_test_sequences, test_labels))
model.summary()
# Plot Loss and Accuracy

acc = history.history['acc']
val_acc = history.history['val_acc']
loss = history.history['loss']
val_loss = history.history['val_loss']

epochs = range(len(acc))

plt.plot(epochs, acc, 'bo', label='Training acc')
plt.plot(epochs, val_acc, 'b', label='Validation acc')
plt.title('Training and validation accuracy')
plt.legend()

plt.figure()

plt.plot(epochs, loss, 'bo', label='Training loss')
plt.plot(epochs, val_loss, 'b', label='Validation loss')
plt.title('Training and validation loss')
plt.legend()

plt.show()<jupyter_output><empty_output><jupyter_text>==> Overfitting<jupyter_code>test_sen = ["I hate this movie so much"]
test_seq = tokenizer.texts_to_sequences(test_sen)
padded_test_sen = pad_sequences(test_seq, maxlen=max_length, padding='post', 
                              truncating="post")
padded_test_sen
model.predict(padded_test_sen)
weights = model.layers[0].get_weights()[0]
weights.shape
#https://projector.tensorflow.org/
out_v = io.open('vecs.tsv', 'w', encoding='utf-8')
out_m = io.open('meta.tsv', 'w', encoding='utf-8')

for idx in range(1, vocab_size):
  word = tokenizer.index_word[idx]
  vector = weights[idx]
  out_m.write(word + "\n")
  out_v.write('\t'.join([str(x) for x in vector]) + "\n")

out_m.close()
out_v.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12305699481865284, 'fraction_numerical': 0.012694300518134714, 'mean_word_length': 5.381818181818182, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 11, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '122237', 'license': 'MIT', 'n_tokens_mistral': 1372, 'n_tokens_neox': 1290, 'n_words': 272, 'path': 'WordEmbedding.ipynb', 'repo_name': 'Phanttan/DeepLearning-Miscellanious', 'size': '3860'}