<jupyter_start><jupyter_text>Romanian Sentence Classification<NAME> [Data Science] Load and install the required modules<jupyter_code>import sys
import os
import warnings
import shutil
from datetime import datetime
import json
DATA_DIR='data'
OUTPUT_DIR='output'

def mount_gdrive():
    global DATA_DIR
    global OUTPUT_DIR
    # Import the library and kaggle config files from gdrive
    GDRIVE_PATH='/content/drive/MyDrive/RESEARCH/'
    if 'google.colab' in sys.modules:
        from google.colab import drive
        import shutil
        drive.mount('/content/drive/')
        sys.path.append(GDRIVE_PATH)
        os.makedirs('/root/.kaggle/', exist_ok=True)
        shutil.copyfile(GDRIVE_PATH + 'kaggle.json', '/root/.kaggle/kaggle.json')
        !chmod 600 '/root/.kaggle/kaggle.json'
        DATA_DIR = os.path.join(GDRIVE_PATH, DATA_DIR)
        OUTPUT_DIR = os.path.join(GDRIVE_PATH, OUTPUT_DIR)

def install_modules():
    !pip install --quiet kaggle
    !pip install --quiet pandas
    !pip install --quiet scikit-learn
    !pip install --quiet scikit-image
    !pip install --upgrade --quiet tensorflow
    !pip install --quiet randomcolor
    !pip install --quiet matplotlib
    !pip install --quiet Pillow
    !pip install --quiet seaborn
    !pip install --quiet tf-models-official
    !pip install --quiet snowballstemmer
    !pip install --quiet nltk
    !pip install --quiet tensorflow-text

mount_gdrive()
#install_modules()
os.makedirs(DATA_DIR, exist_ok=True)
os.makedirs(OUTPUT_DIR, exist_ok=True)
os.makedirs('model_checkpoint', exist_ok=True)

import numpy as np
import pandas as pd
import pickle
import multiprocessing
import sklearn
import scipy
import nltk
nltk.download('punkt')
import tensorflow as tf

CPU_COUNT = multiprocessing.cpu_count()
print(f"Num CPUs Available: {CPU_COUNT}")

#os.environ['CUDA_VISIBLE_DEVICES'] = '-1'
print("Num GPUs Available: ", len(tf.config.list_physical_devices('GPU')))
from tensorflow.python.platform import build_info
print(build_info.build_info)
tf.config.experimental.set_memory_growth(tf.config.list_physical_devices('GPU')[0], True)

BEST_WEIGHTS_TRANSFER='weights_nlp/transformer_weights.save'
TRAIN_LANGUAGE_MODEL = not os.path.exists(BEST_WEIGHTS_TRANSFER)
PROCESS_WIKIPEDIA=False

TRAIN_EPOCHS=60
NO_CLASSES=10
MAX_LENGTH=128
BATCH_SIZE=96<jupyter_output>[nltk_data] Downloading package punkt to
[nltk_data]     C:\Users\suppo\AppData\Roaming\nltk_data...
[nltk_data]   Package punkt is already up-to-date!<jupyter_text>Load my own matmih helper libray<jupyter_code>!git clone "https://github.com/glypher/matmih.git"
!git -C matmih pull

import matmih as mm<jupyter_output>fatal: destination path 'matmih' already exists and is not an empty directory.
git: 'credential-cache' is not a git command. See 'git --help'.<jupyter_text>Process wikipediaDownload romanian wiki: https://dumps.wikimedia.org/rowiki/20210101/rowiki-20210101-pages-articles.xml.bz2Load romanian wikipedia<jupyter_code>if PROCESS_WIKIPEDIA:
    from gensim.corpora import WikiCorpus, Dictionary

    @mm.benchmark
    def load_wiki(path='data/rowiki-20210101-pages-articles.xml.bz2'):
        wiki = WikiCorpus(path, lower=True, token_min_len=1, dictionary=Dictionary())
        wikiDF = pd.DataFrame({'text':wiki.get_texts()})
        wikiDF['sid'] = range(len(wikiDF)) 
        return wikiDF
    wikiDF = load_wiki()
if PROCESS_WIKIPEDIA:
    @mm.benchmark
    def preprocess_df(df, vocab={}):
        preprocess=[]
        if isinstance(df['text'][0], str):
            preprocess= ['lower', 'tokenize']
        pipeline = mm.PreprocessPipeline(df, "romanian", vocab=vocab).process(
            preprocess + ['stem', 'build_vocabulary', 'to_vocabulary_ids'])
        return pipeline.DF, pipeline.VOCAB

    wDF, wVOCAB = preprocess_df(wikiDF)<jupyter_output><empty_output><jupyter_text>VarDial 2021 - Romanian Dialect Identification (RDI)https://www.aclweb.org/anthology/P19-1068.pdfLoad MOROCO dataset<jupyter_code>!git clone "https://github.com/butnaruandrei/MOROCO"
os.chdir('./MOROCO')
from MOROCO.loadDataSet import loadMOROCODataSamples

def loadDF(data_type):
    _cat_labels=['None', 'culture', 'finance', 'politics', 'science', 'sports', 'tech']
    ids, samples, labels, categories = loadMOROCODataSamples(data_type)
    df = pd.DataFrame({'sid':ids, 'text':samples, 'target':labels, 'dialect':labels, 'category':categories})
    df['target'] = df['target'].apply(lambda i: 'MD' if i == 1 else 'RO').astype("category")
    df['dialect'] = df['dialect'].apply(lambda i: 'MD' if i == 1 else 'RO').astype("category")
    df['category'] = df['category'].apply(lambda i: _cat_labels[i]).astype("category")
    return df

trainDF = loadDF('train')
valDF = loadDF('validation')
testDF = loadDF('test')
os.chdir('..')<jupyter_output>fatal: destination path 'MOROCO' already exists and is not an empty directory.<jupyter_text>Only choose the cultural categoryI limit the information available to only the culture category to be able to train faster on a limited number of samples<jupyter_code>trainDF = trainDF[trainDF['category']=='culture']
valDF = valDF[valDF['category']=='culture']
testDF = testDF[testDF['category']=='culture']<jupyter_output><empty_output><jupyter_text>Romanian sentence data setLoad the data and plot the class histograms in training a validation set<jupyter_code>def show_random_classes(df):
    import random
    from datetime import datetime
    random.seed(datetime.now())
    for cl in df['target'].cat.categories:
        class_df = df[df['target'] == cl]
        idx = random.randrange(len(class_df))
        print(f"Class: {class_df['target'].iloc[idx]} Sentence: {class_df['text'].iloc[idx]}")

mm.PlotBuilder().create_subplots(1,2, (13, 6)).create_histograms(
    [trainDF['target'], valDF['target']],
    ['Class histogram TRAIN', 'Class histogram VALIDATION']).show()

show_random_classes(trainDF)<jupyter_output><empty_output><jupyter_text>Data PrunningRemove invalid entries from the training set and plot histograms the text lengthMost lengths are below 128<jupyter_code>def plot_text_length(df, title, pb=None, per_class=True, no_bins=10):
    data = []
    titles = []
    if per_class:
        for id in range(0, NO_CLASSES):
            df_class = df[df['target'] == id]
            lengths = np.array(df_class['text'].str.len())
            data.append((lengths, 'auto'))
    
            titles.append(f'{title} class {id}')
    else:
        lengths = np.array(df['text'].str.len())
        data = [(lengths, 'auto')]
        titles = [title]

    if pb is None:
        pb = mm.PlotBuilder()
    
    pb.create_histograms(data, titles)


print(f'BEFORE Filtering TRAIN={len(trainDF)} VAL={len(valDF)} TEST={len(testDF)}')
# filter the train and validation set for nan text
trainDF['text'] = trainDF['text'].replace(pd.NA, "")
trainDF['text'] = trainDF['text'].replace("", pd.NA)
trainDF = trainDF.dropna(inplace=False)

valDF['text'] = valDF['text'].replace(pd.NA, "")
valDF['text'] = valDF['text'].replace("", pd.NA)
valDF = valDF.dropna(inplace=False)

testDF['text'] = testDF['text'].replace(pd.NA, "")

pb = mm.PlotBuilder().create_subplots(1, 3, (20, 5))
plot_text_length(trainDF, title="TRAIN", pb=pb, per_class=False)
plot_text_length(valDF, title="VALIDATION", pb=pb, per_class=False)
plot_text_length(testDF, title="TEST", pb=pb, per_class=False)
pb.show()<jupyter_output>BEFORE Filtering TRAIN=1484 VAL=404 TEST=404<jupyter_text>Unsupervised model pre-processingAdd all of the train, validation and test data together and preprare it to build my own custom language model.Do text pre-processing by tokenizing each sentence, lowering, stemming, removing punctuation and building a dictionary of word->id identifiers.Plot the histogram of words in each sentence. Notice sentence length is under 128 words<jupyter_code>all_data = pd.concat([trainDF, valDF, testDF]).reset_index()
preprocessor = mm.PreprocessPipeline(all_data, 'romanian', max_words=128, min_words=5, min_word_count=5).process([
    'lower', 'split_sentences', 'tokenize', 'remove_punctuation',
    'build_vocabulary', 'to_vocabulary_ids', 'filter_rows'])

all_data = preprocessor.DF
VOCAB = preprocessor.VOCAB
VOCAB_R = {v:k for k, v in VOCAB.items()}
VOCAB_SIZE = max(list(VOCAB.values()))

print(f"Vocabulary size: {VOCAB_SIZE} \t\tData Size: {len(all_data)}")

mm.PlotBuilder().create_subplots(1,1, (13, 6)).create_histograms(
    [all_data['text'].apply(len)], ['Number of words after preprocessing']).show()<jupyter_output>Vocabulary size: 8289 		Data Size: 29542<jupyter_text>TF DatasetsHelper tensorflow dataset classes to be used in the tensorflow transformer model.There are 2 different datasets models:- MaskedTrainDataset - my unsupervised tensorflow dataset that uses the masking technique described in the Bert paper for unsupervised training of the language model: randomly mask 15% of the data and for 10% of these 15% replace it by a random word- Dataset - the supervised datased used after the language model training for fine tuning the transfer learning modelAll data is padded to 128 so that we can use batches in our training which requires the input data to have the same shape<jupyter_code>class MaskedTrainDataset(tf.data.Dataset):
    def __new__(cls, df, vocab_size, filter_words, max_size=128):
        def _resize(text, pad_value=0):
            text = np.copy(text)
            text.resize(max_size)
            return text
        df = df[['text', 'target']].copy()
        # get only specific words that we can mask
        df['index'] = df['text'].apply(filter_words)
        # set the length of the index vector
        df['length'] = df['index'].apply(lambda s: len(s) if len(s) < max_size else max_size)
        df['index'] = df['index'].apply(_resize)
        # resize the data to the desired shape will place 0 for padding
        df['text'] = df['text'].apply(_resize)
        
        plain_ds = tf.data.Dataset.from_tensor_slices((tf.cast(np.stack(df['text']), tf.int32),
                                                       tf.cast(df['length'], tf.int32),
                                                       tf.cast(np.stack(df['index']), tf.int32)))
        shuffle_size = (int)(len(df) / CPU_COUNT) + 1
        
        @tf.function
        def _mask_data(text, length, can_mask_indices):
            encoded = tf.identity(text)

            # 15% BERT masking
            mask_length = (int)(tf.cast(length, tf.float32) * 0.15)

            mask_length = mask_length if mask_length > 1 else 1
            if length > 0:
                mask_index = tf.random.uniform(shape=[mask_length], minval=0, maxval=length, dtype=tf.int32)

                # get the actual position from the wanted indices
                mask_index = tf.gather(can_mask_indices, mask_index)
                mask_index = tf.reshape(mask_index, [mask_length, 1])
            else:
                mask_index = tf.convert_to_tensor([[0]])

            # set the sample weight to only consider masked items
            sample_weights = tf.zeros_like(text)
            ones = tf.ones([mask_length], dtype=tf.int32)
            sample_weights = tf.tensor_scatter_nd_update(sample_weights, mask_index, ones)
                
            # mask the words
            zeros = tf.zeros([mask_length], dtype=tf.int32)
            encoded = tf.tensor_scatter_nd_update(encoded, mask_index, zeros)

            return (encoded, text, sample_weights)
    
        @tf.function
        def shard_mask_ds(i):
            if i == 0:
                tf.print('masking data...')
            shard_ds = plain_ds.shard(CPU_COUNT, i)
            return shard_ds.map(_mask_data).shuffle(shuffle_size)
    
        # since cache() is not used this will be remasked each time on a new epoch
        return tf.data.Dataset.range(0, CPU_COUNT).interleave(
                    lambda i: shard_mask_ds(i), cycle_length=CPU_COUNT,
                    block_length=16, num_parallel_calls=CPU_COUNT).batch(BATCH_SIZE)


class Dataset(tf.data.Dataset):
    def __new__(cls, df, test=False, max_size=128):
        def _resize(text):
            # resize the data to the desired shape will place 0 for padding
            text = np.copy(text)
            text.resize(max_size)
            return text
        
        df = df[['text', 'target']].copy()
        df['text'] = df['text'].apply(_resize)
        
        if test:
            return tf.data.Dataset.from_tensor_slices((tf.cast(np.stack(df['text']), tf.int32))).batch(
                BATCH_SIZE).prefetch(tf.data.experimental.AUTOTUNE).cache()
        
        return tf.data.Dataset.from_tensor_slices((tf.cast(np.stack(df['text']), tf.int32),
                                                   tf.cast(df['target'].cat.codes, tf.int32))).shuffle(
            len(df)).batch(BATCH_SIZE).prefetch(tf.data.experimental.AUTOTUNE)


@mm.benchmark
def sample_masked_data(df):
    for enc, orig, sw in MaskedTrainDataset(df, len(VOCAB)+1,
                                            lambda wids: [min(i, 127) for i,w in enumerate(wids) if len(VOCAB_R.get(w, "")) > 6]).take(1):
        enc = enc.numpy()
        orig = orig.numpy()
        sample_weight = sw.numpy()
        for i in range(1):
            print(' '.join([VOCAB_R.get(w, '<p>') for w in orig[i]]))
            print('-----------------------')
            print(' '.join([VOCAB_R.get(w, '<m>') for w in enc[i]]))
            mask_index = orig[i] != enc[i]
            print(f"Masked words: {'    '.join([VOCAB_R.get(w, '<p>') for w in orig[i][mask_index]])}")
            print('-----------------------')
            print(sample_weight[i])

sample_masked_data(all_data)<jupyter_output>masking data...
evenimentul a inclus <p> <p> moderne <p> de bibliotecă <p> publicului despre procesul de modernizare a bibliotecilor din ne și startul oficial al campaniei naționale de comunicare <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p> <p>
-----------------------
evenimentul a inclus <m> <m> moderne <m> de <m> <m> publicului despre procesul de modernizare a bibliotecilor din ne și startul oficial al campaniei naționale de comunicare <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m> <m>[...]<jupyter_text>Unsupervised language modelUsed to train the transformer model in an unsupervised way by feeding it a masked input sentence and predicting the words that have been masked. To do this the loss will only take into account the cross entropy error of only the masked words, all the words that have not been masked will not be considered.This is done by using for through the MaskedTrainDataset by constructing for each sentence/sample a sample_weight that will be passed in the training step. This wil be used to compute the loss for each sample and has a 0 weight for all words that have **not** been masked and 1 for the masked words.<jupyter_code>class LanguageModel(tf.keras.Model):
    def __init__(self, inputs, output):
        super(LanguageModel, self).__init__(inputs, output)
        self._loss = tf.keras.losses.SparseCategoricalCrossentropy(reduction=tf.keras.losses.Reduction.NONE)
        self._metrics = [tf.keras.metrics.Mean(name="loss"), tf.keras.metrics.Accuracy()]

    def train_step(self, inputs):
        features, labels, sample_weight = inputs

        with tf.GradientTape() as tape:
            predictions = self(features, training=True)
            loss = self._loss(labels, predictions, sample_weight=sample_weight)

        trainable_vars = self.trainable_variables
        gradients = tape.gradient(loss, self.trainable_variables)
        self.optimizer.apply_gradients(zip(gradients, self.trainable_variables))

        self._metrics[0].update_state(loss, sample_weight=sample_weight)
        self._metrics[1].update_state(labels, tf.math.argmax(predictions, axis=2), sample_weight=sample_weight)

        return {"loss": self._metrics[0].result(), "accuracy": self._metrics[1].result()}

    @property
    def metrics(self):
        return self._metrics<jupyter_output><empty_output><jupyter_text>Transformer modelThe transformer model to be used uses the Google's official model garden implementation of transformers https://github.com/tensorflow/models/tree/master/officialThen EncodersStack is Google's implementation of a Bert type Encoder using the self-attention transformer type followed by several dense layers - https://github.com/tensorflow/models/blob/master/official/nlp/transformer/transformer.pyThe configuration for this EncoderStack is taken from the Google's default Bert implementation with 12 attention heads.The big difference is that Bert is using 12 EncoderStack layers and I am using only 2.To be able to train it usupervised I am using the above custom loss in LanguageModel<jupyter_code>from official.nlp.transformer import attention_layer, transformer

class MyEncoderStack(transformer.EncoderStack):
    """Subclass transformer to be able to save self-attention output"""
    def __init__(self, params, save_attention=False):
        super(MyEncoderStack, self).__init__(params)
        self._save_attention = save_attention
        self._attention = []
    
    @property
    def attention(self):
        return self._attention

    
    def build(self, input_shape):
        super(MyEncoderStack, self).build(input_shape)
        if self._save_attention:
            for layer in self.layers:
                self._attention.append(tf.Variable(tf.zeros(input_shape[1:]), dtype=tf.float32))
    
    def call(self, encoder_inputs, attention_bias, inputs_padding, training):
        """Override the call method to save the attention if needed"""
        for n, layer in enumerate(self.layers):
            self_attention_layer = layer[0]
            feed_forward_network = layer[1]
        
            with tf.name_scope("layer_%d" % n):
                with tf.name_scope("self_attention"):
                    encoder_inputs = self_attention_layer(
                        encoder_inputs, attention_bias, training=training)
                    if self._save_attention:
                        self._attention[n].assign(encoder_inputs[0])
                with tf.name_scope("ffn"):
                    encoder_inputs = feed_forward_network(
                        encoder_inputs, training=training)
                
        return self.output_normalization(encoder_inputs)
from official.nlp.transformer import model_utils

class TransformerModel(mm.TensorModel):
    def __init__(self, **hyper_params):
        self._hyper_params = hyper_params.copy()
        
        self._inputs = tf.keras.layers.Input(shape=(128), dtype=tf.int32, name='words_vocab_ids')
        
        inputs_padding =  model_utils.get_padding(self._inputs)
        attention_bias = model_utils.get_padding_bias(self._inputs)
        
        layer = tf.keras.layers.Embedding(input_dim=hyper_params['vocab_size']+1,
                                          output_dim=hyper_params['embedding_size'],
                                          input_length=128, mask_zero=False)(self._inputs)

        # transformer encoder using self_attention from google model garden
        layer = MyEncoderStack({'num_hidden_layers':hyper_params['encoders'],
                                'hidden_size':hyper_params['embedding_size'],
                                'filter_size':hyper_params['filter_size'], 'relu_dropout':0.3,
                                'layer_postprocess_dropout':0.3,
                                'num_heads':hyper_params['attention_heads'], 'attention_dropout':0.1},
                              hyper_params.get('save_attention', False))(layer, attention_bias, inputs_padding)

        layer = tf.keras.layers.Dense(hyper_params['vocab_size']+1, activation="softmax",
                                      name='word_predictor')(layer)

        super(TransformerModel, self).__init__(LanguageModel(self._inputs, layer),
                                               checkpoint=True)

        self._model.compile(
             optimizer=hyper_params['optimizer']())
        
    def load_weights(self):
        # load initial checkpoints if they are available
        if self._hyper_params.get('load_initial_weights', None) is not None:
            initial_weights = self._hyper_params['load_initial_weights']
            if not isinstance(initial_weights, str):
                initial_weights = 'initial_weights_' + '_'.join(map(lambda x:str(x), self._hyper_params['filter_size']))
                initial_weights = os.path.join(MODEL_CHECKPOINT, initial_weights)

            print(f'Loading initial weights {initial_weights}')
            self._model.load_weights(initial_weights)

        return self

    def train(self, trainDF):
        callbacks = []
        callbacks += [tf.keras.callbacks.ModelCheckpoint(self.best_weights_path, monitor='loss',
                                                         verbose=0, save_best_only=True,
                                                         save_weights_only=True, mode='auto')]
        
        trainDF = MaskedTrainDataset(trainDF, self._hyper_params['vocab_size']+1,
                                    self._hyper_params['mask_words'])

        history = self._model.fit(trainDF,
                                  epochs=self._hyper_params['train_epochs'],
                                  callbacks=callbacks)

        return mm.ModelHistory(self._hyper_params, history.history)
    
    def predict(self, testDF):
        scores = self._model.predict(Dataset(testDF, test=True))
        return np.argmax(scores, axis=-1) + 1, np.array(scores)<jupyter_output><empty_output><jupyter_text>Transfer learning modelThe model that will be fine tuned as a supervised model on the train dataset contains the previous learned transformer language model with a dense 10 softmax classifier as the last layer.  Training the Language modelI am here training the transformer model unsupervised with the masked training dataset and custom lossThe training takes a lot of time (8 hours on an Amazon spot P3 instance)The default weighted adam optimizer is used from Google's model official since is the optimizer they use to train Bert.<jupyter_code>import official.nlp.optimization
from functools import partial

def create_optimizer(df_size, lr=3e-5):
    steps_per_epoch = (int)(df_size / BATCH_SIZE)
    num_train_steps = steps_per_epoch * TRAIN_EPOCHS
    num_warmup_steps = int(0.1*num_train_steps)

    return official.nlp.optimization.create_optimizer(lr,
                                          num_train_steps=num_train_steps,
                                          num_warmup_steps=num_warmup_steps,
                                          optimizer_type='adamw')



@mm.benchmark
def train_language_model():
    TRAIN_EPOCHS=30
    
    lm_hyper_lookup = mm.HyperParamsLookup(lambda hyper_params: TransformerModel(**hyper_params),
                                           lambda hist: np.min(hist.history('loss', mm.DataType.TRAIN)))

    tf.keras.backend.clear_session()
    lm_hyper_lookup.grid_search(all_data, log=True, save_checkpoints=True,
                                train_epochs=[TRAIN_EPOCHS],
                                mask_words=[ # mask random words
                                           lambda wids: range(len(wids)),
                                             # mask random words with length greater than 5
                                           lambda wids: [min(i, 127) for i,w in enumerate(wids) if len(VOCAB_R.get(w, "")) > 5],
                                             # mask random words with length lower than 4
                                           lambda wids: [min(i, 127) for i,w in enumerate(wids) if len(VOCAB_R.get(w, "")) < 4]
                                           ],
                                vocab_size=[VOCAB_SIZE],
                                encoders=[4],
                                attention_heads=[12],
                                embedding_size=[768],
                                filter_size=[3072],
                                optimizer=[partial(create_optimizer, len(all_data), 3e-4)])
    return lm_hyper_lookup
   
    
if TRAIN_LANGUAGE_MODEL:
    lm_hyper_lookup = train_language_model()
    mm.StoreLocal('data/lm_hyper_lookup.bin').save(lm_hyper_lookup)
    BEST_WEIGHTS_TRANSFER = lm_hyper_lookup.best_checkpoint
if TRAIN_LANGUAGE_MODEL:
    ev = mm.ModelEvaluation(lm_hyper_lookup.history)
    ev.plot_history('Training history', ['accuracy', 'loss'])

    model = TransformerModel(**lm_hyper_lookup.best_params,
                            load_initial_weights=BEST_WEIGHTS_TRANSFER)
    model.load_weights()

    for masked, orig, sample_weight in MaskedTrainDataset(all_data[0:BATCH_SIZE],
                                                          len(VOCAB)+1, lm_hyper_lookup.best_params['mask_words']).take(1):
        output = model._model(masked)
        result = tf.math.argmax(model._model(masked), axis=-1).numpy()
        masked = masked.numpy()
        orig = orig.numpy()
        for i in range(1):
            index = orig[i] != 0
            mask_index = orig[i] != masked[i]
            print('------------ORIGINAL------------')
            print(' '.join([VOCAB_R.get(w, '<p>') for w in orig[i][index]]))
            print('-------------MASKED-------------')
            print(' '.join([VOCAB_R.get(w, '<m>') for w in masked[i][index]]))
            print('----------PREDICTED MASK--------')
            print('    '.join([VOCAB_R.get(w, '<p  >') for w in result[i][mask_index]]))<jupyter_output><empty_output><jupyter_text>Attention heatmap<jupyter_code>tf.get_logger().setLevel('ERROR')
def plot_attention(model, text:str):
    df = pd.DataFrame({'sid':[0], 'text':[text], 'target':[0]})
    df_text = mm.PreprocessPipeline(df, 'romanian', vocab=VOCAB).process([
        'lower', 'tokenize', 'remove_punctuation']).DF
    df = mm.PreprocessPipeline(df_text, 'romanian', vocab=VOCAB).process([
        'build_vocabulary', 'to_vocabulary_ids']).DF

    _,_ = model.predict(df)

    # get all attention output data from the encoder
    try:
        encoder = model._model.layers[-3]
        encoder_layers = encoder.layers
    except:
        encoder = model._model.layers[-2]
        encoder_layers = encoder.layers
    attention = [np.mean(encoder.attention[layer].numpy(), axis=1) for layer in range(len(encoder_layers))]
    size = len(df_text['text'][0])
    attention = [a[:size] for a in attention]

    d = pd.DataFrame(data=attention, columns=df_text['text'][0])
    mm.PlotBuilder().create_subplots(1,1, (18,4)).create_heatmap(
        d, "Encoder attention per layer", vmin=0, vmax=1.0).show()

for history, checkpoint in zip(lm_hyper_lookup.history.histories, lm_hyper_lookup.checkpoints):
    model = TransformerModel(**history.model_params,
                            load_initial_weights=checkpoint,
                            save_attention=True)
    model.load_weights()

    plot_attention(model, "Pe câmpul de luptă, Ştefan cel $NE$ nu avea milă faţă de duşmanii săi, foarte mulţi soldaţi fiind ucişi sau luaţi în robie de către domnitorul moldovean")<jupyter_output>Loading initial weights ./model_checkpoint/best_epoch_weights_1a3f7fd8-43fa-47ce-a931-532b9c9bbab6.h5<jupyter_text>Supervised trainingDo the same text-preprocessing as in the unsupervised training and use the vocabulary from that.Train the model on the original train-validation split<jupyter_code>trainDFwid = mm.PreprocessPipeline(trainDF, 'romanian', vocab=VOCAB, max_words=128, min_words=5, min_word_count=5).process(
    ['lower', 'split_sentences', 'tokenize', 'remove_punctuation',
    'build_vocabulary', 'to_vocabulary_ids', 'filter_rows']).DF
valDFwid = mm.PreprocessPipeline(valDF, 'romanian', vocab=VOCAB, max_words=128, min_words=5, min_word_count=5).process(
    ['lower', 'split_sentences', 'tokenize', 'remove_punctuation',
    'build_vocabulary', 'to_vocabulary_ids']).DF
testDFwid = mm.PreprocessPipeline(testDF, 'romanian', vocab=VOCAB, max_words=128, min_words=5, min_word_count=5).process(
    ['lower', 'split_sentences', 'tokenize', 'remove_punctuation',
    'build_vocabulary', 'to_vocabulary_ids']).DF

print(f"SPLIT DATA SIZE  TRAIN:{len(trainDFwid)}  VALIDATION:{len(valDFwid)}  TEST:{len(testDFwid)}")
class SplitPredictionCallback(tf.keras.callbacks.Callback):
    def __init__(self, valDF, splitvalDF, model):
        super(SplitPredictionCallback, self).__init__()
        self._model = model
        self._valDF = valDF
        self._splitvalDF = splitvalDF

    def on_epoch_end(self, epoch, logs={}):
        targets, _ = self._model.predict(self._splitvalDF)
        accuracy = mm.Model.accuracy(np.stack(self._valDF['target'].cat.codes), targets)
        self._model._true_accuracy.append(accuracy)
        print(f"TRUE mode accuracy VALIDATION {accuracy}")

class ClassifierModel(TransformerModel):
    def __init__(self, **hyper_params):
        super(ClassifierModel, self).__init__(**hyper_params)
        
        self.transfer_layer = self._model.layers[-2]

        # replace the last layer
        layer = tf.keras.layers.GlobalAveragePooling1D()(self.transfer_layer.output)

        classifier = tf.keras.layers.Dense(NO_CLASSES, activation='softmax',
                                           name="CLASS_OUTPUT")(layer)
        
        new_model = tf.keras.Model(inputs=self._inputs, outputs=classifier)
        
        # compile the model and initialize the weights
        new_model.compile(
             optimizer=hyper_params['optimizer'](),
             loss="sparse_categorical_crossentropy",
             metrics="accuracy")
        
        # load the previous trained weights
        self.load_weights()
        # replace the model
        self._model = new_model
        self._true_accuracy = []

    def train(self, data):
        trainDF = data[0]
        valDF = data[1]
        fullValDF = data[2]

        # save the weights of the best epoch to disk
        callbacks = []
        callbacks += [SplitPredictionCallback(fullValDF, valDF, self)]
        callbacks += [tf.keras.callbacks.ModelCheckpoint(self.best_weights_path, monitor='val_accuracy',
                                                         verbose=0, save_best_only=True,
                                                         save_weights_only=True, mode='auto')]
        if valDF is not None:
            callbacks += [tf.keras.callbacks.EarlyStopping(monitor='val_accuracy',
                                                           min_delta=0.001, mode='max', patience=5)]
        
        trainDF = Dataset(trainDF)
        if valDF is not None:
            valDF = Dataset(valDF)

        history = self._model.fit(trainDF,
                                  validation_data=valDF,
                                  epochs=self._hyper_params['train_epochs'],
                                  callbacks=callbacks)

        history = history.history.copy()
        history['true_accuracy'] = None
        history['val_true_accuracy'] = self._true_accuracy
        return mm.ModelHistory(self._hyper_params, history)

    def predict(self, testDF):
        split_scores = self._model.predict(Dataset(testDF, test=True))
        
        # reduce targets and scores per sid
        sid_scores = {}
        for i, (sid, text) in testDF[['sid', 'text']].iterrows():
            lst = sid_scores.get(sid, [])
            lst.append(split_scores[i])
            sid_scores[sid] = lst

        scores = []
        targets = []
        last_sid = None
        for sid in testDF['sid']:
            if sid == last_sid:
                continue
            last_sid = sid
            sample_score = np.sum(np.array(sid_scores[sid]), axis=0)
            sample_score = scipy.special.softmax(sample_score)

            targets.append(np.argmax(sample_score, axis=-1))
            scores.append(sample_score)
    
        return np.array(targets), np.array(scores)
TRAIN_EPOCHS=5

BEST_WEIGHTS_TRANSFER = lm_hyper_lookup.best_checkpoint

train_lookup = mm.HyperParamsLookup(lambda hyper_params: ClassifierModel(**hyper_params),
                                    lambda hist: np.max(hist.history('accuracy', mm.DataType.VALIDATION)))

train_lookup.grid_search((trainDFwid, valDFwid, valDF), log=True, save_checkpoints=True,
                         train_epochs=[TRAIN_EPOCHS],
                         vocab_size=[len(VOCAB)],
                         encoders=[4],
                         attention_heads=[12],
                         embedding_size=[768],
                         filter_size=[3072],
                         optimizer=[partial(create_optimizer, len(all_data), 3e-4)],
                         load_initial_weights=lm_hyper_lookup.checkpoints)

mm.StoreLocal('data/train_lookup.bin').save(train_lookup)<jupyter_output>Loading initial weights ./model_checkpoint/best_epoch_weights_1a3f7fd8-43fa-47ce-a931-532b9c9bbab6.h5
Epoch 1/5
197/197 [==============================] - 147s 723ms/step - loss: 1.1903 - accuracy: 0.5015 - val_loss: 0.5943 - val_accuracy: 0.6758
TRUE mode accuracy VALIDATION 0.8366336633663366
Epoch 2/5
197/197 [==============================] - 142s 723ms/step - loss: 0.4465 - accuracy: 0.7900 - val_loss: 0.6911 - val_accuracy: 0.6816
TRUE mode accuracy VALIDATION 0.8564356435643564
Epoch 3/5
197/197 [==============================] - 139s 706ms/step - loss: 0.1752 - accuracy: 0.9322 - val_loss: 0.9306 - val_accuracy: 0.6785
TRUE mode accuracy VALIDATION 0.8762376237623762
Epoch 4/5
197/197 [==============================] - 139s 706ms/step - loss: 0.0461 - accuracy: 0.9842 - val_loss: 1.5234 - val_accuracy: 0.7047
TRUE mode accuracy VALIDATION 0.8589108910891089
Epoch 5/5
197/197 [==============================] - 139s 706ms/step - loss: 0.0201 - accuracy: 0.9925 - val_loss: 2.1016 [...]<jupyter_text>Training resultsPlot the training and validation accuracy and loss of the above training.Dotted line represents training metric. On the right you can see that the Validation loss and that the model strat to overfit quite early - this kind of means that the model has too much capacity and too little regularization<jupyter_code>ev = mm.ModelEvaluation(train_lookup.history)
ev.plot_history('Training history', ['accuracy', 'loss', 'true_accuracy'])<jupyter_output><empty_output><jupyter_text>Test results.Plot the model metrics, ROC curve and confusion matrix for the original train-validation split and test<jupyter_code># Load the model again using the epoch with the highest validation accuracy
def load_model(create_func, best_params, best_checkpoint, save_attention=False):
    hyper_params = best_params.copy()
    hyper_params['load_initial_weights'] = None
    hyper_params['save_attention'] = save_attention
    print(f"Model using {hyper_params}")

    model = create_func(**hyper_params)
    model._hyper_params['load_initial_weights'] = best_checkpoint
    model.load_weights()
    return model

def show_metrics(model, df, split_df, title, plot=True):
    targets, scores = model.predict(split_df)
    print(f"{title} accuracy: {mm.Model.accuracy(df['target'].cat.codes, targets)}")
    if plot:
        targets = pd.Categorical(['MD' if i==0 else 'RO' for i in targets], categories=df['target'].cat.categories)
        #print(targets==df['target'])
        pb = mm.PlotBuilder().create_subplots(1, 2, (16,8))
        pb.create_confusion_matrix(
            df['target'], targets, df['target'].cat.categories)

        pb.create_roc_curve_multiclass(
            df['target'], scores, df['target'].cat.categories, [True, True]).show()


best_model_train = load_model(ClassifierModel, train_lookup.best_history.model_params, train_lookup.best_checkpoint)
show_metrics(best_model_train, valDF, valDFwid, "VALIDATION")
show_metrics(best_model_train, testDF, testDFwid, "TEST")
for history, checkpoint in zip(train_lookup.history.histories, train_lookup.checkpoints):
    model = load_model(ClassifierModel, history.model_params, checkpoint, save_attention=True)
    plot_attention(model, "Pe câmpul de luptă, Ştefan cel $NE$ nu avea milă faţă de duşmanii săi, foarte mulţi soldaţi fiind ucişi sau luaţi în robie de către domnitorul moldovean")<jupyter_output>Model using {'train_epochs': 5, 'vocab_size': 8289, 'encoders': 4, 'attention_heads': 12, 'embedding_size': 768, 'filter_size': 3072, 'optimizer': functools.partial(<function create_optimizer at 0x000002479B4A91F0>, 29542, 0.0003), 'load_initial_weights': None, 'save_attention': True}
Loading initial weights ./model_checkpoint/best_epoch_weights_ba58a55f-9727-43f2-90a4-d1b3fc91c0da.h5
{'fraction_non_alphanumeric': 0.12312215119885643, 'fraction_numerical': 0.016991665992394208, 'mean_word_length': 2.9757666738151403, 'pattern_counts': {'":': 2, '<': 227, '<?xml version=': 0, '>': 230, 'https://': 6, 'lorem ipsum': 0, 'www.': 1, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 75, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '366293', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 12051, 'n_tokens_neox': 11349, 'n_words': 2654, 'path': 'romanian_bert_wiki.ipynb', 'repo_name': 'glypher/romanian_bert', 'size': '37082'}