<jupyter_start><jupyter_text>The IMDb DatasetThe IMDb dataset consists of sentences from movie reviews and human annotations of their sentiment. The task is to predict the sentiment of a given sentence. We use the two-way (positive/negative) class split, and use only sentence-level labels.<jupyter_code>from IPython.display import display, Markdown
with open('../../doc/env_variables_setup.md', 'r') as fh:
    content = fh.read()
display(Markdown(content))<jupyter_output><empty_output><jupyter_text>Import Packages<jupyter_code>import tensorflow as tf
import tensorflow_datasets

from tensorflow.keras.utils import to_categorical

from transformers import (
    BertConfig,
    BertTokenizer,
    TFBertModel,
    TFBertForSequenceClassification,
    glue_convert_examples_to_features,
    glue_processors
)

from sklearn.metrics import confusion_matrix, accuracy_score
from sklearn.metrics import classification_report

import matplotlib.pyplot as plt

import math
import numpy as np
import os
import time
from datetime import timedelta
import shutil
from datetime import datetime
import pickle

# new
import re
from keras.models import Sequential, load_model<jupyter_output><empty_output><jupyter_text>Check configuration<jupyter_code>print(tf.version.GIT_VERSION, tf.version.VERSION)
print(tf.keras.__version__)
gpus = tf.config.list_physical_devices('GPU')
if len(gpus)>0:
    for gpu in gpus:
        print('Name:', gpu.name, '  Type:', gpu.device_type)
else:
    print('No GPU available !!!!')<jupyter_output>No GPU available !!!!<jupyter_text>Define Paths<jupyter_code># note: these need to be specified in the config.sh file
try:
    data_dir=os.environ['PATH_DATASETS']
except KeyError:
    print('missing PATH_DATASETS')
try:   
    tensorboard_dir=os.environ['PATH_TENSORBOARD']
except KeyError:
    print('missing PATH_TENSORBOARD')
try:   
    savemodel_dir=os.environ['PATH_SAVE_MODEL']
except KeyError:
    print('missing PATH_SAVE_MODEL')<jupyter_output><empty_output><jupyter_text>Import local packages<jupyter_code>import preprocessing.preprocessing as pp
import utils.model_metrics as mm
import importlib
importlib.reload(pp);
importlib.reload(mm);<jupyter_output><empty_output><jupyter_text>Loading a data from Tensorflow Datasets<jupyter_code>data, info = tensorflow_datasets.load(name="imdb_reviews",
                            data_dir=data_dir,
                            as_supervised=True,
                            with_info=True)
# IMDb specific:
data_valid = data['test'].take(1000)

# trying to create a true validation data set for after the computation
#data_valid_ext = data['test'].take(2000)
#data_valid = data_valid_ext.take(1000)<jupyter_output><empty_output><jupyter_text>Checking basic info from the metadata<jupyter_code>info
pp.print_info_dataset(info)<jupyter_output>Labels:
      ['neg', 'pos']

Number of label:
      2

Structure of the data:
      dict_keys(['text', 'label'])

Number of entries:
   Train dataset: 25000
   Test dataset:  25000
--> validation dataset not defined<jupyter_text>Checking basic info from the metadata<jupyter_code>data
data.keys()
# only works for glue-compatible datasets
try:
    pp.print_info_data(data['train'])
except AttributeError:
    print('data format incompatible')<jupyter_output># Structure of the data:

   <DatasetV1Adapter shapes: ((), ()), types: (tf.string, tf.int64)>

# Output shape of one entry:
   (TensorShape([]), TensorShape([]))

# Output types of one entry:
   (tf.string, tf.int64)

# Output typesof one entry:
   (<class 'tensorflow.python.framework.ops.Tensor'>, <class 'tensorflow.python.framework.ops.Tensor'>)
 

# Shape of the data:

   (25000, 2)
   ---> 25000 entries
   ---> 2 dim
           [text            / label           ]
           [()              / ()              ]
           [|S709           / |S1             ]


# Examples of data:
array([b"This was an absolutely terrible movie. Don't be lured in by <NAME> or <NAME>. Both are great actors, but this must simply be their worst role in history. Even their great acting could not redeem this movie's ridiculous storyline. This movie is an early nineties US propaganda piece. The most pathetic scenes were those when the Columbian rebels were making their cases for revo[...]<jupyter_text>Define parameters of the model<jupyter_code># changes: had to eliminate all lines concerning a test data set because we only have train and valid


# define parameters
#BATCH_SIZE_TRAIN = 32
#BATCH_SIZE_TEST = 32
#BATCH_SIZE_VALID = 64
#EPOCH = 2
#TOKENIZER = 'bert-base-multilingual-uncased'
#MAX_LENGTH = 512

# extract parameters
size_train_dataset = info.splits['train'].num_examples

# the size for the validation data set has been manually computed according to the function 
# pp.print_info_data because the test set has been manually split above
size_valid_dataset = np.shape(np.array(list(data_valid.as_numpy_iterator())))[0]
number_label = info.features["label"].num_classes

# computer parameter
#STEP_EPOCH_TRAIN = math.ceil(size_train_dataset/BATCH_SIZE_TRAIN)
#STEP_EPOCH_VALID = math.ceil(size_valid_dataset/BATCH_SIZE_VALID)


#print('Dataset size:          {:6}/{:6}'.format(size_train_dataset, size_valid_dataset))
#print('Batch size:            {:6}/{:6}'.format(BATCH_SIZE_TRAIN, BATCH_SIZE_VALID))
#print('Step per epoch:        {:6}/{:6}'.format(STEP_EPOCH_TRAIN, STEP_EPOCH_VALID))
#print('Total number of batch: {:6}/{:6}'.format(STEP_EPOCH_TRAIN*(EPOCH+1), STEP_EPOCH_VALID*(EPOCH+1)))<jupyter_output><empty_output><jupyter_text>Additional steps for the IMDb dataset specifically  Cleaning<jupyter_code>def preprocess_reviews(reviews):
    #REPLACE_NO_SPACE = re.compile("[.;:!\'?,\"()\[\]]")
    REPLACE_WITH_SPACE = re.compile("(<br\s*/><br\s*/>)|(\-)|(\/)")
    
    #ae, oe, ue => only for GERMAN data
    #REPLACE_UMLAUT_AE = re.compile("(ae)")
    #REPLACE_UMLAUT_OE = re.compile("(oe)")
    #REPLACE_UMLAUT_UE = re.compile("(ue)")
    
    #reviews = [REPLACE_NO_SPACE.sub("", line[0].decode("utf-8").lower()) for line in np.array(list(reviews.as_numpy_iterator()))]
    reviews = [REPLACE_WITH_SPACE.sub(" ", line[0].decode("utf-8")) for line in np.array(list(reviews.as_numpy_iterator()))]# for line in reviews]
    #reviews = [REPLACE_UMLAUT_AE.sub("ä", line[0]) for line in reviews]
    #reviews = [REPLACE_UMLAUT_OE.sub("ö", line[0]) for line in reviews]
    #reviews = [REPLACE_UMLAUT_UE.sub("ü", line[0]) for line in reviews]
    
    return reviews
reviews_train_clean = preprocess_reviews(data['train'])
reviews_valid_clean = preprocess_reviews(data_valid)
# calculate the number of characters
x = []
for i in reviews_valid_clean:
    x.append(len(i))
    
sum(x)
# divide into two batches
batch_1 = reviews_valid_clean[:500]
batch_2 = reviews_valid_clean[500:]<jupyter_output><empty_output><jupyter_text>Translating the Validation Dataset<jupyter_code># do this for 3 examples first
# step 1: save data in the right format (.txt, .tsv or html)
with open('en_batch_2.txt', 'w') as f:
    for item in batch_2:
#    for item in reviews_valid_clean[:3]:
        f.write("%s\n\n\n" % item)

# step 2: upload to storage bucket 1 (os.environ['BUCKET_NAME'])
# gsutil cp /home/vera_luechinger/proj_multilingual_text_classification/notebook/00-Test/en_batch_2.txt gs://os.environ['BUCKET_NAME']/

# step 3: translate in storage and store in bucket 2 (os.environ['BUCKET_NAME']_translation: must be empty before the translation process begins)


# batch translation

from google.cloud import translate
import time

def batch_translate_text(
    input_uri="gs://"+os.environ['BUCKET_NAME']+"/en_batch_2.txt",
    output_uri="gs://"+os.environ['BUCKET_NAME_TRANSLATION']+"/",
    project_id=os.environ['PROJECT_ID']
):
    """Translates a batch of texts on GCS and stores the result in a GCS location."""

    client = translate.TranslationServiceClient()


    location = "us-central1"
    # Supported file types: https://cloud.google.com/translate/docs/supported-formats
    gcs_source = {"input_uri": input_uri}

    input_configs_element = {
        "gcs_source": gcs_source,
        "mime_type": "text/plain"  # Can be "text/plain" or "text/html".
    }
    gcs_destination = {"output_uri_prefix": output_uri}
    output_config = {"gcs_destination": gcs_destination}
    parent = client.location_path(project_id, location)

    # Supported language codes: https://cloud.google.com/translate/docs/language
    start_time = time.time()

    operation = client.batch_translate_text(
        parent=parent,
        source_language_code="en",
        target_language_codes=["fr","de"],  # Up to 10 language codes here.
        input_configs=[input_configs_element],
        output_config=output_config)

    print(u"Waiting for operation to complete...")
    response = operation.result(180)
    elapsed_time_secs = time.time() - start_time
    print(u"Execution Time: {}".format(elapsed_time_secs))
    print(u"Total Characters: {}".format(response.total_characters))
    print(u"Translated Characters: {}".format(response.translated_characters))
batch_translate_text()
# step 4: save files in the first bucket
#gsutil cp gs://os.environ['BUCKET_NAME']+_translation/os.environ['BUCKET_NAME']_en_batch_2_fr_translations.txt gs://os.environ['BUCKET_NAME']/batch_2/

de_1_dir = "gs://"+os.environ['BUCKET_NAME']+"/batch_1/"+os.environ['BUCKET_NAME']+"_en_batch_1_de_translations.txt"
from google.cloud import storage
#from config import bucketName, localFolder, bucketFolder

storage_client = storage.Client()
bucket = storage_client.get_bucket(os.environ['BUCKET_NAME'])
#bucket
def download_file(bucketName, file, localFolder):
    """Download file from GCP bucket."""
    #fileList = list_files(bucketName)
    #rand = randint(0, len(fileList) - 1)
    storage_client = storage.Client()
    bucket = storage_client.get_bucket(bucketName)
    blob = bucket.blob(file)
    fileName = blob.name.split('/')[-1]
    blob.download_to_filename(localFolder + fileName)
    return f'{fileName} downloaded from bucket.'
# drop this before pushing

download_file(os.environ['BUCKET_NAME'], "batch_1/"+os.environ['BUCKET_NAME']+"_en_batch_1_fr_translations.txt", "/home/vera_luechinger/data/imdb_reviews/")
download_file(os.environ['BUCKET_NAME'], "batch_1/"+os.environ['BUCKET_NAME']+"_en_batch_1_de_translations.txt", "/home/vera_luechinger/data/imdb_reviews/")
download_file(os.environ['BUCKET_NAME'], "batch_2/"+os.environ['BUCKET_NAME']+"_en_batch_2_fr_translations.txt", "/home/vera_luechinger/data/imdb_reviews/")
download_file(os.environ['BUCKET_NAME'], "batch_2/"+os.environ['BUCKET_NAME']+"_en_batch_2_de_translations.txt", "/home/vera_luechinger/data/imdb_reviews/")
print("")
# step 5: get translated files from storage to use in notebook
with open("/home/vera_luechinger/data/imdb_reviews/"+os.environ['BUCKET_NAME']+"_en_batch_1_de_translations.txt", 'r') as file:
    de_1 = file.readlines()
    
with open("/home/vera_luechinger/data/imdb_reviews/"+os.environ['BUCKET_NAME']+"_en_batch_2_de_translations.txt", 'r') as file:
    de_2 = file.readlines()

with open("/home/vera_luechinger/data/imdb_reviews/"+os.environ['BUCKET_NAME']+"_en_batch_1_fr_translations.txt", 'r') as file:
    fr_1 = file.readlines()

with open("/home/vera_luechinger/data/imdb_reviews/"+os.environ['BUCKET_NAME']+"_en_batch_2_fr_translations.txt", 'r') as file:
    fr_2 = file.readlines()

de = de_1 + de_2
fr = fr_1 + fr_2
de = [item.replace("\n","") for item in de]
fr = [item.replace("\n","") for item in fr]

len(de)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1254551760013872, 'fraction_numerical': 0.011271024796254551, 'mean_word_length': 4.888208269525268, 'pattern_counts': {'":': 5, '<': 49, '<?xml version=': 0, '>': 54, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '16336', 'license': 'Apache-2.0', 'n_tokens_mistral': 3992, 'n_tokens_neox': 3937, 'n_words': 852, 'path': 'notebook/00-Test/08_IMDb_Huggingface_translation.ipynb', 'repo_name': 'tarrade/proj_multilingual_text_classification', 'size': '11556'}