<jupyter_start><jupyter_text>Package Preparation<jupyter_code>!pip install tf_sentencepiece
import tensorflow as tf
import numpy as np
import tensorflow_hub as hub
import tf_sentencepiece
from random import shuffle, choice
import re
import os
import datetime
from functools import reduce
from operator import itemgetter
from google.colab import drive
drive.mount('/content/gdrive')<jupyter_output>Collecting tf_sentencepiece
[?25l  Downloading https://files.pythonhosted.org/packages/72/fe/363d78d29c556d0da642ffe285c2c7573b6a83239a9b0d08d83376c9fbac/tf_sentencepiece-0.1.82.1-py2.py3-none-manylinux1_x86_64.whl (2.8MB)
[K     |████████████████████████████████| 2.8MB 6.5MB/s 
[?25hInstalling collected packages: tf-sentencepiece
Successfully installed tf-sentencepiece-0.1.82.1<jupyter_text>Configurations<jupyter_code>xling_encoding_len = 512
max_seg = 10
level_class_cnt = 3

test_percentage = 0.1
validation_percentage = 0.1

dropout_rate = 0.5
eta = 1e-4
hidden_feature_dim = 100
attention_key_dim = 100
gru_feature_dim = 50

batch_size = 512
epochs = 8

label_re = re.compile('(\d+)\.\d+')
sentence_re = re.compile('(?:\.|!|\?)\s')
edu_re = edu_re = re.compile('(?:\.|!|\?|(?:EDU_BREAK))\s')

input_path = '/content/gdrive/My Drive/data_source/milnet/raw_text/gourmet_edus.txt'
model_out_path = '/content/gdrive/My Drive/data_source/milnet/results/food_xling_edu_c3.h5'
log_out_dir = '/content/gdrive/My Drive/data_source/milnet/log/'

out_test_path = '/content/gdrive/My Drive/data_source/milnet/raw_text/organic_edus.txt'

sample_amount = 0
with open(input_path) as in_file:
    sample_amount = len(in_file.read().split('\n\n')) - 1

with open(out_test_path) as in_file:
    out_test_samples = [*range(len(in_file.read().split('\n\n')) - 1)]

sample_indices = [*range(sample_amount)]
shuffle(sample_indices)
train_samples = sample_indices[0:int(sample_amount * (1 - test_percentage - validation_percentage))]
validation_samples = sample_indices[int(sample_amount * (1 - test_percentage - validation_percentage)): int(sample_amount * (1 - test_percentage))]
test_samples = sample_indices[int(sample_amount * (1 - test_percentage)):]<jupyter_output><empty_output><jupyter_text>Data Preloading<jupyter_code># Building the graph of XLING
g = tf.Graph()
with g.as_default():
    text_input = tf.placeholder(dtype=tf.string, shape=[None])
    en_de_embed = hub.Module("https://tfhub.dev/google/universal-sentence-encoder-xling/en-de/1")
    embedded_text = en_de_embed(text_input)
    init_op = tf.group([tf.global_variables_initializer(), tf.tables_initializer()])
g.finalize()
session = tf.Session(graph=g)
session.run(init_op)
''' Add segment padding for document encoding.

Args:
    doc_encoding (np.array): The encoding of document. Number of segment in the document is undefined.
    max_seg (int): The maximum number of segments in a document. If the actual segments is not enough, 
        the result will be padded, otherwise it will be truncated.

Returns:
    (np.array): The padded or truncated matrix of document encoding.

'''
def __pad_doc_encoding(doc_encoding, max_seg):
    if doc_encoding.shape[0] > max_seg:
        return doc_encoding[:max_seg]
    elif doc_encoding.shape[0] < max_seg:
        topad_len = max_seg - doc_encoding.shape[0]
        pad_width = [(0, 0) if i != 0 else (0, topad_len) for i in range(len(doc_encoding.shape))]
        return np.pad(doc_encoding, pad_width, 'constant', constant_values=0)
    else:
        return doc_encoding

''' Mapping the rating labels to sentiment levels.

Args:
    raw_label (int): The rating label in the raw dataset.

Returns:
    (int): The sentiment level for the raw label.
'''
def __label_map(raw_label):
    if raw_label == 1 or raw_label == 2:
        return 0
    elif raw_label == 3:
        return 1
    else:
        return 2

''' Remove the samples with rating label 2 and 4 in the dataset.

This method will help to maximize the distance between the classes.

Args:
    feature_array (np.array): The feature numpy array.
    label_array (np.array): The label numpy array, the labels should not be mapped yet.

Returns:
    (np.array, np.array): The feature numpy array and the label numpy array.
'''
def __balance_data(feature_array, label_array):
    to_balance_indices = np.concatenate([np.where(label_array == 2)[0], np.where(label_array == 4)[0]])
    return np.delete(feature_array, to_balance_indices, axis=0), np.delete(label_array, to_balance_indices, axis=0)

''' Generate the batch data for `fit_generator`. The raw text will be encoded by XLING in this method.

Args:
    input_path (str): The path of the .hdf5 file.
    sample_indices (int): The indices of the samples used in the text file.
    segment_re (re): The regex for segmenting the document.
    label_re (re): The regex for recognizing the labels.
    batch_size (int): The batch size.
    max_seg (int): The maximum number of segmentation in a document.
    xling_len (int): The length of the XLING embedding vector.
    epochs (int): The expected epochs of the generator.
    use_balance (boolean): Whether removing the samples with rating 2 and 4.
'''
def data_generator(input_path, sample_indices, segment_re, label_re, 
                   batch_size=batch_size, max_seg=max_seg, xling_len=xling_encoding_len, epochs=epochs, use_balance=True):
    global session, embedded_text, text_input
    with open(input_path) as in_file:
        file_content = [*itemgetter(*sample_indices)(in_file.read().split('\n\n'))]
        for _ in range(epochs):
            shuffle(file_content)
            feature_cache, label_cache = [], []
            batch_index = 0
            for sample in file_content:
                label_cache.append(sample.split('\n')[0])
                feature_cache.append([*filter(lambda x: len(x) > 1, segment_re.split(' '.join(sample.split('\n')[1:])))])
                batch_index += 1
                if batch_index == batch_size:
                    len_lst = [*map(len, feature_cache)]
                    batch_features = session.run(embedded_text, feed_dict={text_input: reduce(lambda x, y: x + y, feature_cache)})
                    label_array = np.array([np.array([int(label_re.findall(l)[0])]) for l in label_cache])
                    feature_array = np.zeros((batch_size, max_seg, xling_len))
                    for index, length in enumerate(len_lst):
                        feature_array[index] = __pad_doc_encoding(np.array(batch_features[:length]), max_seg)
                        batch_features = batch_features[length:]
                    feature_array = np.array(feature_array)
                    if use_balance:
                        feature_array, label_array = __balance_data(feature_array, label_array)
                    yield feature_array, np.array([np.array([__label_map(l[0])]) for l in label_array])
                    feature_cache, label_cache = [], []
                    batch_index = 0
# Container for layers, since multiple instance learning requires reusing classifier for segments.
shared_sublayer_cache = {}

''' Execute `sublayer` for every tensor in `layer_in` splitted along first dimension.

Args:
    layer_in (Tensor): The input tensor.
    sublayer (func): The function applied to all splitted tensors.
    args (dict): The arguments of sublayer

'''
def branch_execute(layer_in, sublayer, args={}):
    instance_cnt = layer_in.shape[1]
    sliced_inputs = [tf.keras.layers.Lambda(lambda x: x[:,i])(layer_in) 
                     for i in range(instance_cnt)]
    branch_layers = [sublayer(**{**{'layer_in': sliced_inputs[i]}, **args}) 
                     for i in range(instance_cnt)]
    expand_layer = tf.keras.layers.Lambda(lambda x: tf.keras.backend.expand_dims(x, axis=1))
    expanded_layers = [expand_layer(branch_layers[i]) for i in range(instance_cnt)]
    concated_layer = tf.keras.layers.Concatenate(axis=1)(expanded_layers)
    return concated_layer

def __seg_classifier_layer_share(layer_in, class_cnt, dropout_rate, eta):
    global shared_sublayer_cache
    if 'shared_seg_classifier_sublayers' not in shared_sublayer_cache:
        shared_sublayer_cache['shared_seg_classifier_sublayers'] = {
            'drop_out_layer': tf.keras.layers.Dropout(
                dropout_rate
            ),
            'dense_layer': tf.keras.layers.Dense(
                units=class_cnt,
                activation='softmax',
                kernel_regularizer=tf.keras.regularizers.l2(eta),
                bias_regularizer=tf.keras.regularizers.l2(eta)
            )
        }
    shared_layers = shared_sublayer_cache['shared_seg_classifier_sublayers']
    drop_out_layer = shared_layers['drop_out_layer'](layer_in)
    dense_layer = shared_layers['dense_layer'](drop_out_layer)
    return dense_layer

def __attention_layer_share(layer_in, attention_key_dim, dropout_rate, eta):
    global shared_sublayer_cache
    if 'shared_attention_sublayers' not in shared_sublayer_cache:
        shared_sublayer_cache['shared_attention_sublayers'] = {
            'drop_out_layer': tf.keras.layers.Dropout(
                dropout_rate
            ),
            'dense_layer': tf.keras.layers.Dense(
                units=attention_key_dim, 
                activation='tanh',
                kernel_regularizer=tf.keras.regularizers.l2(eta),
                bias_regularizer=tf.keras.regularizers.l2(eta)
            ),
            'nobias_dense_layer': tf.keras.layers.Dense(
                units=1, 
                use_bias=False, 
                bias_regularizer=tf.keras.regularizers.l2(eta)
            )
        }
    shared_layers = shared_sublayer_cache['shared_attention_sublayers']
    drop_out_layer = shared_layers['drop_out_layer'](layer_in)
    dense_layer = shared_layers['dense_layer'](drop_out_layer)
    nobias_dense_layer = shared_layers['nobias_dense_layer'](dense_layer)
    return nobias_dense_layer

def bidirectional_gru_layer(layer_in, gru_feature_dim):
    bidirectional_layer = tf.keras.layers.Bidirectional(
        tf.keras.layers.GRU(gru_feature_dim, return_sequences=True)
    )(layer_in)
    return bidirectional_layer

def merge_layer(layer_in, class_cnt, eta):
    dot_layer = tf.keras.layers.Dot(axes=1)(layer_in)
    flatten_layer = tf.keras.layers.Flatten()(dot_layer)
    dense_layer = tf.keras.layers.Dense(
        units=class_cnt, 
        activation='softmax',
        kernel_regularizer=tf.keras.regularizers.l2(eta),
        bias_regularizer=tf.keras.regularizers.l2(eta)
    )(flatten_layer)
    return dense_layer

def performance_judge(model, generator, class_cnt):
    eps = np.finfo(float).eps
    accuracy, precisions, recalls, f1s = [], [], [], []
    for i, (features, labels) in enumerate(generator):
        predicted = model.predict(features)
        precisions.append([])
        recalls.append([])
        f1s.append([])
        contingency_table = np.zeros((class_cnt, class_cnt))
        for index in range(features.shape[0]):
            contingency_table[int(labels[index][0])][np.argmax(predicted[index])] += 1
        accuracy.append(np.trace(contingency_table) / features.shape[0])
        for index in range(class_cnt):
            precisions[i].append(contingency_table[index][index] / (np.sum(contingency_table[:, index]) + eps))
            recalls[i].append(contingency_table[index][index] / (np.sum(contingency_table[index, :]) + eps))
            f1s[i].append(2 * precisions[i][-1] * recalls[i][-1] / ((precisions[i][-1] + recalls[i][-1]) + eps))
    precisions = [float(sum(l))/len(l) for l in zip(*precisions)]
    recalls = [float(sum(l))/len(l) for l in zip(*recalls)]
    f1s = [float(sum(l))/len(l) for l in zip(*f1s)]
    print('Accuracy:', round(reduce(lambda x, y: x + y, accuracy) / len(accuracy), 3))
    for index in range(class_cnt):
        print('_____ Class', index, '_____')
        print('Precision\t', round(precisions[index], 3))
        print('Recall\t\t', round(recalls[index], 3))
        print('F1 Score\t', round(f1s[index], 3))
print('Constructing Model ...', end='')

model_input = tf.keras.Input((max_seg, xling_encoding_len))

biglu_layer = bidirectional_gru_layer(
    model_input, 
    gru_feature_dim=50
)

attention_layer = branch_execute(
    biglu_layer, 
    sublayer=__attention_layer_share, 
    args={
        'attention_key_dim': 100,
        'dropout_rate': 0.5,
        'eta': 1e-4
    }
)

softmaxed_attention_layer = tf.keras.layers.Softmax(
    axis=1
)(attention_layer)

classification_layer = branch_execute(
    model_input, 
    sublayer=__seg_classifier_layer_share, 
    args={
        'class_cnt': level_class_cnt,
        'dropout_rate': 0.5,
        'eta': 1e-4
    }
)

merge_layer = merge_layer(
    [softmaxed_attention_layer, classification_layer],
    class_cnt=level_class_cnt,
    eta=1e-4
)

model = tf.keras.Model(model_input, merge_layer)

print('\rModel Constructed. Compiling ...', end='')

model.compile(
    optimizer=tf.keras.optimizers.Adam(clipvalue=0.5),
    loss=tf.keras.losses.SparseCategoricalCrossentropy(),
    metrics=['accuracy']
)

print('\rModel Compiled.')

model.summary()
logdir = os.path.join(log_out_dir, datetime.datetime.now().strftime("%Y%m%d_%H%M%S"))
tensorboard_callback = tf.keras.callbacks.TensorBoard(logdir, histogram_freq=0)

model.fit_generator(
    data_generator(input_path, train_samples, edu_re, label_re, use_balance=True),
    validation_data=data_generator(input_path, validation_samples, edu_re, label_re, use_balance=True),
    steps_per_epoch=(sample_amount * (1 - test_percentage - validation_percentage) // batch_size) - 1,
    validation_steps=(sample_amount * (validation_percentage) // batch_size) - 1,
    validation_freq=2,
    epochs=epochs,
    callbacks=[tensorboard_callback]
)

model.save(model_out_path)

print('########## Training Error ##########')
performance_judge(model, data_generator(input_path, train_samples, edu_re, label_re, epochs=1, use_balance=True), level_class_cnt)
print('')
print('############ Test Error ############')
performance_judge(model, data_generator(input_path, train_samples, edu_re, label_re, epochs=1, use_balance=True), level_class_cnt)
print('')
print('####### No Cheat Test Error ########')
performance_judge(model, data_generator(input_path, train_samples, edu_re, label_re, epochs=1, use_balance=False), level_class_cnt)

print(logdir)<jupyter_output>Epoch 1/8
{'fraction_non_alphanumeric': 0.12207194645844953, 'fraction_numerical': 0.013733965421081986, 'mean_word_length': 3.72341126111294, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 13, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '474290', 'license': 'MIT', 'n_tokens_mistral': 4904, 'n_tokens_neox': 4698, 'n_words': 953, 'path': 'model/milnet_xling.ipynb', 'repo_name': 'tpfeifle/MILNET_tensorflow', 'size': '14344'}