<jupyter_start><jupyter_text>**General Work Process**1. Import dataset and preprocess2. Train model3. Test model<jupyter_code>import io
import os
import re
import shutil
import string
import numpy as np
import pandas as pd
import tensorflow as tf

from tensorflow.keras import Sequential, layers, losses
from tensorflow.keras.layers import Dense, Embedding, GlobalAveragePooling1D
from tensorflow.keras.layers import TextVectorization
url = "https://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz"
dataset = tf.keras.utils.get_file("aclImdb_v1.tar.gz", url,
                                  untar=True, cache_dir='.',
                                  cache_subdir='')
dataset_dir = os.path.join(os.path.dirname(dataset), 'aclImdb')
os.listdir(dataset_dir)
# view train data files
train_dir = os.path.join(dataset_dir, 'train')
os.listdir(train_dir)
# clean unnecessary empty folder
remove_dir = os.path.join(train_dir, 'unsup')
shutil.rmtree(remove_dir)
batch_size = 1024
seed = 10

train_data = tf.keras.preprocessing.text_dataset_from_directory(
    'aclImdb/train',
    batch_size=batch_size, 
    validation_split=0.2,
    subset='training', 
    seed=seed)

val_data = tf.keras.preprocessing.text_dataset_from_directory(
    'aclImdb/train', 
    batch_size=batch_size, 
    validation_split=0.2,
    subset='validation', 
    seed=seed)
# sample batch from train data
for text_batch, label_batch in train_data.take(1):
    
    # view the first 5 samples
    for i in range(5):
        print(label_batch[i].numpy(), text_batch.numpy()[i])
AUTOTUNE = tf.data.AUTOTUNE

train_ds = train_data.cache().prefetch(buffer_size=AUTOTUNE)
val_ds = val_data.cache().prefetch(buffer_size=AUTOTUNE)
# Create a custom standardization function to strip HTML break tags '<br />'.
def custom_standardization(input_data):
      lowercase = tf.strings.lower(input_data)
      stripped_html = tf.strings.regex_replace(lowercase, '<br />', ' ')
      return tf.strings.regex_replace(stripped_html,
                                      '[%s]' % re.escape(string.punctuation), '')


# Vocabulary size and number of words in a sequence.
vocab_size = 10000
sequence_length = 100

# Use the text vectorization layer to normalize, split, and map strings to
# integers. Note that the layer uses the custom standardization defined above.
# Set maximum_sequence length as all samples are not of the same length.
vectorize_layer = TextVectorization(
                    standardize=custom_standardization,
                    max_tokens=vocab_size,
                    output_mode='int',
                    output_sequence_length=sequence_length)

# Make a text-only dataset (no labels) and call adapt to build the vocabulary.
text_ds = train_ds.map(lambda x, y: x)
vectorize_layer.adapt(text_ds)
embedding_dim=16

model = Sequential([
  vectorize_layer,
  Embedding(vocab_size, embedding_dim, name="embedding"),
  GlobalAveragePooling1D(),
  Dense(32, activation='relu'),
  Dense(1)
])

tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir="logs")

model.compile(optimizer='adam',
              loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
              metrics=['accuracy'])
model.fit(
    train_ds,
    validation_data=val_ds,
    epochs=20,
    callbacks=[tensorboard_callback])
%load_ext tensorboard
%tensorboard --logdir logs
# get the trained word embeddings
weights = model.get_layer('embedding').get_weights()[0]
vocab = vectorize_layer.get_vocabulary()
vocab[:10]
out_v = io.open('vectors.tsv', 'w', encoding='utf-8')
out_m = io.open('metadata.tsv', 'w', encoding='utf-8')

for index, word in enumerate(vocab):
  if index == 0:
    continue  # skip 0, it's padding.
  vec = weights[index]
  out_v.write('\t'.join([str(x) for x in vec]) + "\n")
  out_m.write(word + "\n")
out_v.close()
out_m.close()<jupyter_output><empty_output><jupyter_text>Test model<jupyter_code># view test data files
test_dir = os.path.join(dataset_dir, 'test')
os.listdir(test_dir)
test_data = tf.keras.preprocessing.text_dataset_from_directory(
    'aclImdb/test')
def vectorize_text(text, label):
    text = tf.expand_dims(text, -1)
    return vectorize_layer(text), label
# sample batch from test data
for test_text_batch, test_label_batch in test_data.take(1):
    
    # view the first 5 samples
    for i in range(5):
        print(test_label_batch[i].numpy(), test_text_batch.numpy()[i])
text_batch, label_batch = next(iter(test_data))
first_review, first_label = text_batch[0], label_batch[0]
print("Review", first_review)
print("Label", test_data.class_names[first_label])
print("Vectorized review", vectorize_text(first_review, first_label))
# the vectorize function is not required to process the test data
# if the vectorize layer included in model

# test_ds = test_data.map(vectorize_text)

# # sample batch from test data
# for test_text_batch, test_label_batch in test_ds.take(1):
#     for i in range(1):
#         print(test_label_batch[i].numpy(), test_text_batch.numpy()[i])
loss, accuracy = model.evaluate(test_data)

print("Loss: ", loss)
print("Accuracy: ", accuracy)
export_model = tf.keras.Sequential([
  model,
  layers.Activation('sigmoid')
])

export_model.compile(
    loss=losses.BinaryCrossentropy(from_logits=False), optimizer="adam", metrics=['accuracy']
)

# Test it with `raw_test_ds`, which yields raw strings
loss, accuracy = export_model.evaluate(test_data)
print(accuracy)
text_batch, label_batch = next(iter(test_data))
first_review, first_label = text_batch[0], label_batch[0]
pred_label = export_model.predict(test_data)
pred_label
pred_label.shape
pred_y = []

for i in range(len(pred_label)):
    pred_y.append(round(pred_label[i][0]))
len(pred_y)
actual_y = []
for tt, ll in test_data:
    for l in ll:
        actual_y.append(l.numpy())
correct = 0
for i in range(len(pred_y)):
    if pred_y[i] == actual_y[i]:
        correct+=1
correct/len(pred_y)*100<jupyter_output><empty_output><jupyter_text>**Analyze my own review**<jupyter_code>my_reviews =["The new movie is popular and awesome",
             "The background music is annoying and too loud",
             "We are very enjoy the movie",
             "Negative comment in internent is hurt people",
             "The smile is very sweat and cute!",
             "The view is so beautiful and attrative",
             ]
export_model.predict(my_reviews)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11461273126371903, 'fraction_numerical': 0.009093759799310128, 'mean_word_length': 4.237274220032841, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '5269', 'license': 'MIT', 'n_tokens_mistral': 2154, 'n_tokens_neox': 2098, 'n_words': 491, 'path': 'Text_Sentiment_Analysis/TextVectorization_layer.ipynb', 'repo_name': 'ee2110/Natural_Language_Processing-NLP-TensorFlow', 'size': '6378'}