<jupyter_start><jupyter_text>Imports and global variables<jupyter_code>import tensorflow as tf
from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint, TensorBoard, LearningRateScheduler
from tensorboard.plugins.hparams import api as hp
from transformers import RobertaTokenizerFast, TFRobertaModel, TFRobertaForSequenceClassification, RobertaForSequenceClassification
import numpy as np
import pandas as pd
import os
import datetime
import torch
import json
DATA_FOLDER = os.path.join(os.getcwd(), "Data")
MODELS_FOLDER = os.path.join(os.getcwd(), "Models")
TF_LOGS_FOLDER = os.path.join(os.getcwd(), "tf_logs")
train_df = pd.read_csv(os.path.join(DATA_FOLDER, "corefx_cleaned_train.csv"))
val_df = pd.read_csv(os.path.join(DATA_FOLDER, "corefx_cleaned_val.csv"))
TRAIN_LEN = len(train_df)
VAL_LEN = len(val_df)
del train_df
del val_df
BATCH_SIZE = 32
MAX_LEN = 128
DROPOUT = 0.3
SEED = 2020<jupyter_output><empty_output><jupyter_text>Create Tensorflow datasets<jupyter_code>train_dataset = tf.data.experimental.CsvDataset(os.path.join(DATA_FOLDER, "corefx_cleaned_train.csv"), [tf.string, tf.int32], header=True)
val_dataset = tf.data.experimental.CsvDataset(os.path.join(DATA_FOLDER, "corefx_cleaned_val.csv"), [tf.string, tf.int32], header=True)
tokenizer = RobertaTokenizerFast.from_pretrained("roberta-base")<jupyter_output><empty_output><jupyter_text>Define transformations to do on datasets<jupyter_code>def tf_py_function(sentences):
    decoded = []
    for sentence in sentences.numpy():
        decoded.append(sentence.decode())
    encoded = tokenizer(decoded, padding='max_length', truncation=True, max_length=MAX_LEN, return_tensors='tf')
    input_ids = encoded["input_ids"]
    attention_mask = encoded["attention_mask"]
    return (input_ids, attention_mask)
# Need to use tf.py_function because tokenizer doesn't operate on tensors
def encode_data(batch_x, batch_y):
    input_ids, attention_mask = tf.py_function(tf_py_function, [batch_x], (tf.int32, tf.int32))
    input_ids.set_shape([None, MAX_LEN])
    attention_mask.set_shape([None, MAX_LEN])
    return (input_ids, attention_mask, batch_y)
def map_to_dict(input_ids, attention_mask, labels):
    return ({"input_ids": input_ids, "attention_mask": attention_mask}, labels)<jupyter_output><empty_output><jupyter_text>Apply transformations and prepare datasets for training<jupyter_code>train_dataset = train_dataset.shuffle(TRAIN_LEN, seed=SEED, reshuffle_each_iteration=True).batch(BATCH_SIZE).map(encode_data, num_parallel_calls=tf.data.experimental.AUTOTUNE).map(map_to_dict, num_parallel_calls=tf.data.experimental.AUTOTUNE).repeat().prefetch(1)
val_dataset = val_dataset.shuffle(VAL_LEN, seed=SEED, reshuffle_each_iteration=True).batch(BATCH_SIZE).map(encode_data, num_parallel_calls=tf.data.experimental.AUTOTUNE).map(map_to_dict, num_parallel_calls=tf.data.experimental.AUTOTUNE).repeat().prefetch(1)<jupyter_output><empty_output><jupyter_text>Create and train TF model<jupyter_code>with open(os.path.join(DATA_FOLDER, 'lookup.json')) as json_file: 
    lookup = json.load(json_file) 
github_model = TFRobertaForSequenceClassification.from_pretrained("roberta-base", num_labels=len(lookup.keys()) // 2, hidden_dropout_prob=DROPOUT)
def schedule(epoch, lr):
  if epoch < 10 or epoch > 40:
    return lr
  else:
    return lr * tf.math.exp(-0.1)
opt = tf.keras.optimizers.Adam(learning_rate=5e-5)
loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
accuracy = tf.keras.metrics.SparseCategoricalAccuracy("accuracy")

github_model.compile(optimizer=opt, loss=loss, metrics=[accuracy])
earlystopping = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=30)
reduceLr = ReduceLROnPlateau(monitor='val_loss', mode='min', verbose=1, patience=5, factor=0.75, min_lr=1e-6)
checkpointer = ModelCheckpoint(filepath= os.path.join( MODELS_FOLDER, "github_model_weights.hdf5"), verbose=1, save_best_only=True)
scheduler = LearningRateScheduler(schedule)
# use tensorboard
log_dir = os.path.join(TF_LOGS_FOLDER, datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))
tensorboard = TensorBoard(log_dir=log_dir)
github_model.fit(train_dataset, validation_data=val_dataset, steps_per_epoch= TRAIN_LEN // BATCH_SIZE, epochs=100, validation_steps= VAL_LEN // BATCH_SIZE, callbacks=[earlystopping, reduceLr, checkpointer, tensorboard, scheduler])<jupyter_output>Epoch 1/100
WARNING:tensorflow:Gradients do not exist for variables [&#39;tf_roberta_for_sequence_classification/roberta/pooler/dense/kernel:0&#39;, &#39;tf_roberta_for_sequence_classification/roberta/pooler/dense/bias:0&#39;] when minimizing the loss.
WARNING:tensorflow:Gradients do not exist for variables [&#39;tf_roberta_for_sequence_classification/roberta/pooler/dense/kernel:0&#39;, &#39;tf_roberta_for_sequence_classification/roberta/pooler/dense/bias:0&#39;] when minimizing the loss.
WARNING:tensorflow:Gradients do not exist for variables [&#39;tf_roberta_for_sequence_classification/roberta/pooler/dense/kernel:0&#39;, &#39;tf_roberta_for_sequence_classification/roberta/pooler/dense/bias:0&#39;] when minimizing the loss.
WARNING:tensorflow:Gradients do not exist for variables [&#39;tf_roberta_for_sequence_classification/roberta/pooler/dense/kernel:0&#39;, &#39;tf_roberta_for_sequence_classification/roberta/pooler/dense/bias:0&#39;] when minimizing the loss.
 1/40 [.................[...]<jupyter_text>Save TF model (Huggingface format and SavedModel format) and create Pytorch model from it<jupyter_code>github_model.load_weights(os.path.join( MODELS_FOLDER, "github_model_weights.hdf5"))
github_model.save_pretrained(os.path.join(MODELS_FOLDER, "tf"))
tokenizer.save_pretrained(os.path.join(MODELS_FOLDER, "tf"))
pytorch_model = RobertaForSequenceClassification.from_pretrained(os.path.join(MODELS_FOLDER, "tf"), from_tf=True)
pytorch_model.save_pretrained(os.path.join(MODELS_FOLDER, "pt"))
tokenizer.save_pretrained(os.path.join(MODELS_FOLDER, "pt"))
callable = tf.function(github_model.call)
concrete_function = callable.get_concrete_function([tf.TensorSpec([None, MAX_LEN], tf.int32, name="input_ids"), tf.TensorSpec([None, MAX_LEN], tf.int32, name="attention_mask")])
tf.saved_model.save(github_model, os.path.join(MODELS_FOLDER, "Serving", "1"), signatures=concrete_function)<jupyter_output>WARNING:tensorflow:From C:\ProgramData\Miniconda3\lib\site-packages\tensorflow\python\training\tracking\tracking.py:111: Model.state_updates (from tensorflow.python.keras.engine.training) is deprecated and will be removed in a future version.
Instructions for updating:
This property should not be used in TensorFlow 2.0, as updates are applied automatically.
WARNING:tensorflow:From C:\ProgramData\Miniconda3\lib\site-packages\tensorflow\python\training\tracking\tracking.py:111: Layer.updates (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.
Instructions for updating:
This property should not be used in TensorFlow 2.0, as updates are applied automatically.
INFO:tensorflow:Assets written to: d:\work\RobertaGithubIssuesClassification\Models\Serving\1\assets<jupyter_text>Try models on test issue<jupyter_code>issue = "AppDomain.SetPrincipalPolicy(PrincipalPolicy.WindowsPrincipal) works only once. Setting the PrincipalPolicy on the current AppDomain to WindowsPrincipal works only for the first thread being started. Any subsequent thread has Thread.CurrentPrincipal evaluated to NULL."
label = "area-System.Security"
encoded_tf = tokenizer(issue, padding='max_length', truncation=True, max_length=MAX_LEN, return_tensors='tf', add_special_tokens=True, return_token_type_ids=False)
encoded_pt = tokenizer(issue, padding='max_length', truncation=True, max_length=MAX_LEN, return_tensors='pt', add_special_tokens=True, return_token_type_ids=False)
github_model = TFRobertaForSequenceClassification.from_pretrained(os.path.join(MODELS_FOLDER, "tf"))
tf_result = github_model.predict([encoded_tf['input_ids'], encoded_tf['attention_mask']])
print(tf_result)
pytorch_model.eval()
pt_result = pytorch_model(input_ids=encoded_pt["input_ids"], attention_mask=encoded_pt["attention_mask"])
print(pt_result)
np.allclose(tf_result[0], pt_result[0].detach().numpy(), rtol=1e-03, atol=1e-08)
lookup[str(np.argmax(tf_result))]<jupyter_output><empty_output><jupyter_text>Convert to ONNX and run inference<jupyter_code>import torch.onnx
pytorch_model.eval()
torch.onnx.export(pytorch_model,               # model being run
                  (encoded_pt["input_ids"], encoded_pt["attention_mask"]),  # model input (or a tuple for multiple inputs)
                  os.path.join(MODELS_FOLDER, "roberta_github_issues.onnx"),   # where to save the model (can be a file or file-like object)
                  export_params=True,        # store the trained parameter weights inside the model file
                  opset_version=12,          # the ONNX version to export the model to
                  do_constant_folding=True,  # whether to execute constant folding for optimization
                  input_names = ['input_ids', 'attention_mask'],   # the model's input names
                  output_names = ['output'], # the model's output names
                  dynamic_axes={'input_ids' : {0 : 'batch_size'},
                                'attention_mask' : {0 : 'batch_size'},
                                'output' : {0 : 'batch_size'}}
                    )
import onnx

onnx_model = onnx.load(os.path.join(MODELS_FOLDER, "roberta_github_issues.onnx"))
onnx.checker.check_model(onnx_model)
import onnxruntime

ort_session = onnxruntime.InferenceSession(os.path.join(MODELS_FOLDER, "roberta_github_issues.onnx"))
ort_session.get_inputs()[1].name
def to_numpy(tensor):
    return tensor.detach().cpu().numpy() if tensor.requires_grad else tensor.cpu().numpy()
ort_inputs = {ort_session.get_inputs()[0].name: to_numpy(encoded_pt["input_ids"]), ort_session.get_inputs()[1].name: to_numpy(encoded_pt["attention_mask"])}
ort_outs = ort_session.run(None, ort_inputs)
print(ort_outs)
print(pt_result)
print(tf_result)
np.allclose(ort_outs, tf_result[0], rtol=1e-03, atol=1e-08)
np.allclose(ort_outs, pt_result[0].detach().numpy(), rtol=1e-03, atol=1e-08)
index = np.argmax(to_numpy(pt_result[0]))
print(f"index : {index}, category : {lookup[str(index)]}")
softmax = torch.nn.functional.softmax(pt_result[0])
for i, value in enumerate(softmax[0]):
    print(f"{lookup[str(i)] : <27} : {value.item() * 100 :.2f}% confidence")<jupyter_output>area-System.Net             : 0.25% confidence
area-Infrastructure         : 0.13% confidence
area-System.ComponentModel  : 0.26% confidence
area-System.Security        : 91.46% confidence
area-System.Runtime         : 0.47% confidence
area-System.IO              : 0.10% confidence
area-System.Xml             : 1.56% confidence
area-System.Collections     : 0.04% confidence
area-System.Threading       : 0.24% confidence
area-System.Reflection      : 0.12% confidence
area-System.Memory          : 0.03% confidence
area-System.Diagnostics     : 0.90% confidence
area-Serialization          : 0.12% confidence
area-System.Drawing         : 0.13% confidence
area-Meta                   : 0.13% confidence
area-System.Data            : 2.50% confidence
area-Microsoft.CSharp       : 0.10% confidence
area-System.Numerics        : 0.09% confidence
area-System.Text            : 0.04% confidence
area-System.Globalization   : 1.04% confidence
area-System.Linq            : 0.10% confidence
area-System.[...]
{'fraction_non_alphanumeric': 0.11969011142061281, 'fraction_numerical': 0.018628133704735376, 'mean_word_length': 5.855011933174224, 'pattern_counts': {'":': 2, '<': 32, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '226134', 'license': 'MIT', 'n_tokens_mistral': 4012, 'n_tokens_neox': 3820, 'n_words': 684, 'path': 'TF_roberta_github_issues.ipynb', 'repo_name': 'Ghisso/RobertaGithubIssuesClassification', 'size': '11488'}