<jupyter_start><jupyter_text>The Stanford Sentiment Treebank The Stanford Sentiment Treebank consists of sentences from movie reviews and human annotations of their sentiment. The task is to predict the sentiment of a given sentence. We use the two-way (positive/negative) class split, and use only sentence-level labels.<jupyter_code>from IPython.display import display, Markdown
with open('../../doc/env_variables_setup.md', 'r') as fh:
    content = fh.read()
display(Markdown(content))<jupyter_output><empty_output><jupyter_text>Import Packages<jupyter_code>import tensorflow as tf
import tensorflow_datasets

from tensorflow.keras.utils import to_categorical

from transformers import (
    BertConfig,
    BertTokenizer,
    XLMRobertaTokenizer,
    TFBertModel,
    TFXLMRobertaModel,
    TFBertForSequenceClassification,
    glue_convert_examples_to_features,
    glue_processors
)

from sklearn.metrics import confusion_matrix, accuracy_score
from sklearn.metrics import classification_report

import matplotlib.pyplot as plt

from google.cloud import storage

import math
import numpy as np
import os
import glob
import time
from datetime import timedelta
import shutil
from datetime import datetime
import pickle
import re<jupyter_output><empty_output><jupyter_text>Check configuration<jupyter_code>print(tf.version.GIT_VERSION, tf.version.VERSION)
print(tf.keras.__version__)
gpus = tf.config.list_physical_devices('GPU')
if len(gpus)>0:
    for gpu in gpus:
        print('Name:', gpu.name, '  Type:', gpu.device_type)
else:
    print('No GPU available !!!!')<jupyter_output>No GPU available !!!!<jupyter_text>Define Paths<jupyter_code>try:
    data_dir=os.environ['PATH_DATASETS']
except KeyError:
    print('missing PATH_DATASETS')
try:   
    tensorboard_dir=os.environ['PATH_TENSORBOARD']
except KeyError:
    print('missing PATH_TENSORBOARD')
try:   
    savemodel_dir=os.environ['PATH_SAVE_MODEL']
except KeyError:
    print('missing PATH_SAVE_MODEL')<jupyter_output><empty_output><jupyter_text>Import local packages<jupyter_code>import preprocessing.preprocessing as pp
import utils.model_metrics as mm
import utils.model_utils as mu
import importlib
importlib.reload(pp);
importlib.reload(mm);
importlib.reload(mu);<jupyter_output><empty_output><jupyter_text>Loading a data from Tensorflow Datasets<jupyter_code>data, info = tensorflow_datasets.load(name='glue/sst2',
                                      data_dir=data_dir,
                                      with_info=True)<jupyter_output>INFO:absl:Load dataset info from /Users/tarrade/tensorflow_datasets/glue/sst2/1.0.0
INFO:absl:Field info.description from disk and from code do not match. Keeping the one from code.
INFO:absl:Field info.citation from disk and from code do not match. Keeping the one from code.
INFO:absl:Reusing dataset glue (/Users/tarrade/tensorflow_datasets/glue/sst2/1.0.0)
INFO:absl:Constructing tf.data.Dataset for split None, from /Users/tarrade/tensorflow_datasets/glue/sst2/1.0.0<jupyter_text>Checking baics info from the metadata<jupyter_code>info
pp.print_info_dataset(info)<jupyter_output>Labels:
      ['negative', 'positive']

Number of label:
      2

Structure of the data:
      dict_keys(['sentence', 'label', 'idx'])

Number of entries:
   Train dataset: 67349
   Test dataset:  1821
   Valid dataset: 872<jupyter_text>Checking baics info from the metadata<jupyter_code>data
data.keys()
pp.print_info_data(data['train'])<jupyter_output># Structure of the data:

   <PrefetchDataset shapes: {idx: (), label: (), sentence: ()}, types: {idx: tf.int32, label: tf.int64, sentence: tf.string}>

# Output shape of one entry:
   {'idx': TensorShape([]), 'label': TensorShape([]), 'sentence': TensorShape([])}

# Output types of one entry:
   {'idx': tf.int32, 'label': tf.int64, 'sentence': tf.string}

# Output typesof one entry:
   {'idx': <class 'tensorflow.python.framework.ops.Tensor'>, 'label': <class 'tensorflow.python.framework.ops.Tensor'>, 'sentence': <class 'tensorflow.python.framework.ops.Tensor'>}
 

# Shape of the data:

   (67349,)
   ---> 67349 entries
   ---> 1 dim
        dict structure
           dim: 3
           [idx       / label     / sentence ]
           [()        / ()        / ()       ]
           [int32     / int64     / bytes    ]


# Examples of data:
{'idx': 16399,
 'label': 0,
 'sentence': b'for the uninitiated plays better on video with the sound '}
{'idx': 1680,
 'label': 0,
 'sentence': b'like a gi[...]<jupyter_text>Define parameters of the model<jupyter_code># models
#MODELS = [(TFBertModel,     BertTokenizer,       'bert-base-multilingual-uncased'),
#          (OpenAIGPTModel,  OpenAIGPTTokenizer,  'openai-gpt'),
#          (GPT2Model,       GPT2Tokenizer,       'gpt2'),
#          (CTRLModel,       CTRLTokenizer,       'ctrl'),
#          (TransfoXLModel,  TransfoXLTokenizer,  'transfo-xl-wt103'),
#          (XLNetModel,      XLNetTokenizer,      'xlnet-base-cased'),
#          (XLMModel,        XLMTokenizer,        'xlm-mlm-enfr-1024'),
#          (DistilBertModel, DistilBertTokenizer, 'distilbert-base-cased'),
#          (RobertaModel,    RobertaTokenizer,    'roberta-base'),
#          (XLMRobertaModel, XLMRobertaTokenizer, 'xlm-roberta-base'),
#         ]
MODELS = [(TFBertModel,         BertTokenizer,       'bert-base-multilingual-uncased'),
          (TFXLMRobertaModel,   XLMRobertaTokenizer, 'jplu/tf-xlm-roberta-base')]
model_index = 0 # BERT
model_class        = MODELS[model_index][0] # i.e TFBertModel
tokenizer_class    = MODELS[model_index][1] # i.e BertTokenizer
pretrained_weights = MODELS[model_index][2] #'i.e bert-base-multilingual-uncased'

# Maxium length, becarefull BERT max length is 512!
MAX_LENGTH = 128

# define parameters
BATCH_SIZE_TRAIN = 32
BATCH_SIZE_TEST = 32
BATCH_SIZE_VALID = 64
EPOCH = 2

# extract parameters
size_train_dataset  = info.splits['train'].num_examples
size_test_dataset   = info.splits['test'].num_examples
size_valid_dataset = info.splits['validation'].num_examples
number_label = info.features["label"].num_classes

# computer parameter
STEP_EPOCH_TRAIN = math.ceil(size_train_dataset/BATCH_SIZE_TRAIN)
STEP_EPOCH_TEST = math.ceil(size_test_dataset/BATCH_SIZE_TEST)
STEP_EPOCH_VALID = math.ceil(size_test_dataset/BATCH_SIZE_VALID)


print('Dataset size:          {:6}/{:6}/{:6}'.format(size_train_dataset, size_test_dataset, size_valid_dataset))
print('Batch size:            {:6}/{:6}/{:6}'.format(BATCH_SIZE_TRAIN, BATCH_SIZE_TEST, BATCH_SIZE_VALID))
print('Step per epoch:        {:6}/{:6}/{:6}'.format(STEP_EPOCH_TRAIN, STEP_EPOCH_TEST, STEP_EPOCH_VALID))
print('Total number of batch: {:6}/{:6}/{:6}'.format(STEP_EPOCH_TRAIN*(EPOCH+1), STEP_EPOCH_TEST*(EPOCH+1), STEP_EPOCH_VALID*(EPOCH+1)))<jupyter_output>Dataset size:           67349/  1821/   872
Batch size:                32/    32/    64
Step per epoch:          2105/    57/    29
Total number of batch:   6315/   171/    87<jupyter_text>Tokenizer and prepare data for BERT<jupyter_code># Define the checkpoint directory to store the checkpoints
pretrained_model_dir=savemodel_dir+'/pretrained_model/'+pretrained_weights
os.makedirs(pretrained_model_dir, exist_ok=True)
# Load tokenizer
tokenizer = tokenizer_class.from_pretrained(pretrained_weights, cache_dir=pretrained_model_dir)
# recap of input dataset
print(data['train'])
print(tf.data.experimental.cardinality(data['train']))
print(tf.data.experimental.cardinality(data['test']))
print(tf.data.experimental.cardinality(data['validation']))
# super slow since looping over all data
#print(len(list(data['train'])))
# Prepare data for BERT
train_dataset = glue_convert_examples_to_features(data['train'], 
                                                  tokenizer, 
                                                  max_length=MAX_LENGTH, 
                                                  task='sst-2')
test_dataset = glue_convert_examples_to_features(data['test'], 
                                                  tokenizer, 
                                                  max_length=MAX_LENGTH, 
                                                  task='sst-2')
valid_dataset = glue_convert_examples_to_features(data['validation'], 
                                                  tokenizer, 
                                                  max_length=MAX_LENGTH, 
                                                  task='sst-2')
# adding the number of entries
if tf.version.VERSION[0:5]=='2.2.0':
    train_dataset=train_dataset.apply(tf.data.experimental.assert_cardinality(tf.data.experimental.cardinality(data['train'])))
    test_dataset=test_dataset.apply(tf.data.experimental.assert_cardinality(tf.data.experimental.cardinality(data['test'])))
    valid_dataset=valid_dataset.apply(tf.data.experimental.assert_cardinality(tf.data.experimental.cardinality(data['validation']))) 
# recap of pre processing dataset
print(train_dataset)
if tf.version.VERSION[0:5]=='2.2.0':
    print(tf.data.experimental.cardinality(train_dataset))
    print(tf.data.experimental.cardinality(test_dataset))
    print(tf.data.experimental.cardinality(valid_dataset))
    # super slow since looping over all data
    #print(len(list(train_dataset)))
else:
    print(size_train_dataset)
    print(size_test_dataset)
    print(size_valid_dataset)<jupyter_output><FlatMapDataset shapes: ({input_ids: (None,), attention_mask: (None,), token_type_ids: (None,)}, ()), types: ({input_ids: tf.int32, attention_mask: tf.int32, token_type_ids: tf.int32}, tf.int64)>
67349
1821
872<jupyter_text>Check the final data<jupyter_code>pp.print_info_data(train_dataset,print_example=False)
pp.print_detail_tokeniser(train_dataset, tokenizer)<jupyter_output>input_ids     ---->    attention_mask    token_type_ids    modified text                 

       101     ---->           1                 1          [ C L S ]                     
     10139     ---->           1                 1          f o r                         
     10103     ---->           1                 1          t h e                         
     18768     ---->           1                 1          u n i                         
     45611     ---->           1                 1          # # n i t i                   
     21096     ---->           1                 1          # # a t e d                   
     17173     ---->           1                 1          p l a y s                     
     16197     ---->           1                 1          b e t t e r                   
     10125     ---->           1                 1          o n                           
     11379     ---->           1                 1          v i d e o                    [...]<jupyter_text>Save data as TFRecord files<jupyter_code># Create directory to save TFRecord files
tfrecord_data_dir=data_dir+'/tfrecord/sst2'
os.makedirs(tfrecord_data_dir, exist_ok=True)
pp.write_tf_data_into_tfrecord(train_dataset,tfrecord_data_dir+'/train_dataset')
pp.write_tf_data_into_tfrecord(test_dataset,tfrecord_data_dir+'/test_dataset')
pp.write_tf_data_into_tfrecord(valid_dataset,tfrecord_data_dir+'/valid_dataset')<jupyter_output><empty_output><jupyter_text>Read data from TFRecord files (sanity check)<jupyter_code># TFRecords encode and store data
train_files = tf.data.TFRecordDataset(tfrecord_data_dir+'/train_dataset.tfrecord')
test_files = tf.data.TFRecordDataset(tfrecord_data_dir+'/test_dataset.tfrecord')
valid_files = tf.data.TFRecordDataset(tfrecord_data_dir+'/valid_dataset.tfrecord')
train_dataset2 = train_files.map(pp.parse_tfrecord_glue_files)
test_dataset2 = test_files.map(pp.parse_tfrecord_glue_files)
valid_dataset2 = valid_files.map(pp.parse_tfrecord_glue_files)
# adding the number of entries
if tf.version.VERSION[0:5]=='2.2.0':
    train_dataset2=train_dataset2.apply(tf.data.experimental.assert_cardinality(train_dataset2.reduce(0, lambda x, _: x + 1).numpy()))
    test_dataset2=test_dataset2.apply(tf.data.experimental.assert_cardinality(test_dataset2.reduce(0, lambda x, _: x + 1).numpy()))
    valid_dataset2=valid_dataset2.apply(tf.data.experimental.assert_cardinality(valid_dataset2.reduce(0, lambda x, _: x + 1).numpy()))
if tf.version.VERSION[0:5]=='2.2.0':
    print(tf.data.experimental.cardinality(train_dataset2))
    print(tf.data.experimental.cardinality(test_dataset2))
    print(tf.data.experimental.cardinality(valid_dataset2))
else:
    print(train_dataset2.reduce(0, lambda x, _: x + 1).numpy())
    print(test_dataset2.reduce(0, lambda x, _: x + 1).numpy())
    print(valid_dataset2.reduce(0, lambda x, _: x + 1).numpy())
pp.print_info_data(train_dataset2,print_example=False)
pp.print_detail_tokeniser(train_dataset2, tokenizer)
train_dataset
train_dataset2
for i in train_dataset2:
    print(i)
    break
for i in train_dataset:
    print(i)
    break<jupyter_output>({'input_ids': <tf.Tensor: shape=(128,), dtype=int32, numpy=
array([  101, 10139, 10103, 18768, 45611, 21096, 17173, 16197, 10125,
       11379, 10171, 10103, 14127,   102,     0,     0,     0,     0,
           0,     0,     0,     0,     0,     0,     0,     0,     0,
           0,     0,     0,     0,     0,     0,     0,     0,     0,
           0,     0,     0,     0,     0,     0,     0,     0,     0,
           0,     0,     0,     0,     0,     0,     0,     0,     0,
           0,     0,     0,     0,     0,     0,     0,     0,     0,
           0,     0,     0,     0,     0,     0,     0,     0,     0,
           0,     0,     0,     0,     0,     0,     0,     0,     0,
           0,     0,     0,     0,     0,     0,     0,     0,     0,
           0,     0,     0,     0,     0,     0,     0,     0,     0,
           0,     0,     0,     0,     0,     0,     0,     0,     0,
           0,     0,     0,     0,     0,     0,     0,     0,     0,
           0,     0,     0,  [...]
{'fraction_non_alphanumeric': 0.11333043855840208, 'fraction_numerical': 0.03343465045592705, 'mean_word_length': 2.67331206804891, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 64, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 40, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '578741', 'license': 'Apache-2.0', 'n_tokens_mistral': 4771, 'n_tokens_neox': 4487, 'n_words': 916, 'path': 'notebook/02-Preprocessing/01_SST2_Huggingface_preprocesing.ipynb', 'repo_name': 'tarrade/proj_multilingual_text_classification', 'size': '13818'}