<jupyter_start><jupyter_text>To run the code, please setup the anaconda environment by running the following code.```bashconda env create -f env-gpu.yamlconda activate ML```<jupyter_code>import numpy as np
import json_lines

from keras_xlnet.backend import keras
from keras_xlnet import PretrainedList, get_pretrained_paths
from keras_xlnet import Tokenizer
from keras_bert.layers import Extract
from keras_xlnet import load_trained_model_from_checkpoint, ATTENTION_TYPE_BI

from sklearn.model_selection import train_test_split<jupyter_output><empty_output><jupyter_text>Hyper-parameters<jupyter_code>EPOCH = 5
BATCH_SIZE = 20
SEQ_LEN = 128 # max length of sequence
LR = 4e-6<jupyter_output><empty_output><jupyter_text>Use the pretrained modelDownload from [google](https://storage.googleapis.com/xlnet/released_models/cased_L-12_H-768_A-12.zip)Here it will download it automatically<jupyter_code>paths = get_pretrained_paths(PretrainedList.en_cased_base)<jupyter_output><empty_output><jupyter_text>Model definition<jupyter_code>def get_model():
    model = load_trained_model_from_checkpoint(
        config_path=paths.config,
        checkpoint_path=paths.model,
        batch_size=BATCH_SIZE,
        memory_len=256,
        target_len=SEQ_LEN,
        in_train_phase=False,
        attention_type=ATTENTION_TYPE_BI,
    )

    last = Extract(index=-1, name='Extract')(model.output)
    dense = keras.layers.Dense(units=768, activation='tanh', name='Dense')(last)
    dropout = keras.layers.Dropout(rate=0.1, name='Dropout')(dense)
    output = keras.layers.Dense(units=2, activation='softmax', name='Softmax')(dropout)
    model = keras.models.Model(inputs=model.inputs, outputs=output)
    # model.summary()

    model.compile(
        optimizer=keras.optimizers.Adam(lr=LR),
        loss='sparse_categorical_crossentropy',
        metrics=['sparse_categorical_accuracy'],
    )
    return model<jupyter_output><empty_output><jupyter_text>Data loader<jupyter_code>def load_data(tokenizer: Tokenizer, path, SEQ_LEN, text_label='trans_en', target_label='voted_up'):
    tokens, classes = [], []
    with open(path, 'rb') as reader:
        for line in json_lines.reader(reader):
            encoded = tokenizer.encode(line[text_label])[:SEQ_LEN - 1]
            encoded = [tokenizer.SYM_PAD] * (SEQ_LEN - 1 - len(encoded)) + encoded + [tokenizer.SYM_CLS]
            tokens.append(encoded)
            classes.append(int(line[target_label]))
    tokens, classes = np.array(tokens), np.array(classes)
    return tokens, classes

def get_X_array(X):
    segments = np.zeros_like(X)
    segments[:, -1] = 1
    lengths = np.zeros_like(X[:, :1])
    return [X, segments, lengths]
def model_train(model, target_label='voted_up', model_name='xlnet_voted_up'):
    X, y = load_data(tokenizer, 'data/reviews_112_trans-en.jl',
        SEQ_LEN=SEQ_LEN, target_label=target_label)
    X_train, X_test, y_train, y_test = train_test_split(X,
        y, test_size=0.1, random_state=42)

    mcp_save = keras.callbacks.ModelCheckpoint("result/"+model_name+'.best.h5',
        save_best_only=True, monitor='val_sparse_categorical_accuracy', mode='max')

    model.fit(
        get_X_array(X_train),
        y_train,
        epochs=EPOCH,
        batch_size=BATCH_SIZE,
        validation_split=0.1,
        callbacks=[keras.callbacks.EarlyStopping(monitor='val_loss', patience=2), mcp_save]
    )

    model.save_weights("result/"+model_name+".h5")
tokenizer = Tokenizer(paths.vocab)<jupyter_output><empty_output><jupyter_text>XLNet Early accessTraining process<jupyter_code>model = get_model()
model_train(model, 'early_access', 'xlnet_early_access')<jupyter_output>WARNING:tensorflow:From /users/pgrad/geto/anaconda3/envs/ML/lib/python3.8/site-packages/keras_transformer_xl/memory.py:72: calling Layer.add_update (from tensorflow.python.keras.engine.base_layer) with inputs is deprecated and will be removed in a future version.
Instructions for updating:
`inputs` is now automatically inferred
Epoch 1/5
203/203 [==============================] - 181s 890ms/step - loss: 0.4136 - sparse_categorical_accuracy: 0.8664 - val_loss: 0.2987 - val_sparse_categorical_accuracy: 0.9133
Epoch 2/5
203/203 [==============================] - 178s 879ms/step - loss: 0.3288 - sparse_categorical_accuracy: 0.8867 - val_loss: 0.3054 - val_sparse_categorical_accuracy: 0.9156
Epoch 3/5
203/203 [==============================] - 160s 789ms/step - loss: 0.2415 - sparse_categorical_accuracy: 0.9059 - val_loss: 0.3286 - val_sparse_categorical_accuracy: 0.9089<jupyter_text>Evaluation<jupyter_code>import json
from sklearn.metrics import roc_auc_score, roc_curve, confusion_matrix, accuracy_score
import matplotlib.pyplot as plt
def plot_roc(pred, y):
    fpr, tpr, thresholds = roc_curve(y, pred)
    try:
        auc = roc_auc_score(y, pred)
    except ValueError:
        auc = "undefined"

    fig, ax = plt.subplots(1, figsize=(8,8))
    ax.plot(fpr, tpr, color='red')
    ax.plot([0,1], [0,1], color='black', linestyle='--')
    ax.set_title(f"AUC: {auc}")
model.load_weights("result/xlnet_early_access.best.h5")
X, y = load_data(tokenizer, 'data/reviews_112_trans-en.jl',
        SEQ_LEN=SEQ_LEN, target_label='early_access')
preds = model.predict(get_X_array(X), verbose=True, batch_size=BATCH_SIZE)
result_max = preds.argmax(axis=-1)
tn, fp, fn, tp = confusion_matrix(y, result_max).ravel()

print('Confusion matrix:')
print('[{}, {}]'.format(tp, fp))
print('[{}, {}]'.format(fn, tn))
print('Accuracy: %.4f' % accuracy_score(y, result_max))
plot_roc(preds[:,1], y)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1388340807174888, 'fraction_numerical': 0.029237668161434978, 'mean_word_length': 5.060869565217391, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '169263', 'license': 'MIT', 'n_tokens_mistral': 2117, 'n_tokens_neox': 1975, 'n_words': 339, 'path': 'appendix_xlnet-early-access.ipynb', 'repo_name': 'comword/TCD20CS4CS7-MLFinal', 'size': '5575'}