<jupyter_start><jupyter_text>Train test split<jupyter_code>df_train, df_test = train_test_split(
    data_sample, 
    train_size=0.8, 
    random_state=RANDOM_SEED, 
    shuffle=True,
    stratify=data_sample['target'])
df_val, df_test = train_test_split(
    df_test, 
    train_size=0.5, 
    random_state=RANDOM_SEED, 
    shuffle=True, 
    stratify=df_test['target'])
print(df_train.shape, df_val.shape, df_test.shape)
print(df_train.target.value_counts(normalize = True))<jupyter_output>1    0.541162
0    0.458838
Name: target, dtype: float64<jupyter_text>Train model<jupyter_code>model_dir = "D:/Users/Nicholas/Projects/BERT_pretrained/biobert-base-cased-v1.1"
model_config = {
    'n_classes':2, 
    'add_linear':[512,256], 
    'attn_bias':False, 
    'freeze_layer_count':8
}
trainer = training.Trainer(model_dir, model_config, epochs=4)
for name, param in trainer.model.named_parameters():
    print(name, param.requires_grad)
trainer.fit(df_train, df_val)<jupyter_output>Epoch 1 / 4
--------------------
Batch Train loss: 0.6929     Class 1 prop: 0.4375     ACC: 0.4375     F1: 0.5909     AUC: 0.3058
Batch Train loss: 0.6932     Class 1 prop: 0.5000     ACC: 0.5000     F1: 0.6522     AUC: 0.4896
Batch Train loss: 0.6938     Class 1 prop: 0.5312     ACC: 0.5625     F1: 0.6957     AUC: 0.7066
Batch Train loss: 0.6889     Class 1 prop: 0.4062     ACC: 0.6875     F1: 0.6875     AUC: 0.7274
Batch Train loss: 0.6934     Class 1 prop: 0.5312     ACC: 0.7500     F1: 0.8000     AUC: 0.8648
Batch Train loss: 0.6938     Class 1 prop: 0.5312     ACC: 0.6875     F1: 0.7619     AUC: 0.7754
Batch Train loss: 0.6932     Class 1 prop: 0.5312     ACC: 0.7812     F1: 0.7742     AUC: 0.7791
Batch Train loss: 0.6941     Class 1 prop: 0.5625     ACC: 0.7188     F1: 0.7907     AUC: 0.7931
Batch Train loss: 0.6937     Class 1 prop: 0.5625     ACC: 0.6250     F1: 0.7391     AUC: 0.7620
Batch Train loss: 0.6914     Class 1 prop: 0.5000     ACC: 0.7188     F1: 0.7273     AUC: 0.73[...]<jupyter_text>plot history<jupyter_code>history = trainer.history
epochs = len(history['train_loss'])
plt.plot(list(range(epochs)), history['train_loss'], label = 'train loss')
plt.plot(list(range(epochs)), history['val_loss'], label = 'val loss')
plt.legend()
plt.plot(list(range(epochs)), history['train_f1'], label = 'train f1')
plt.plot(list(range(epochs)), history['val_f1'], label = 'val f1')
plt.legend()
plt.plot(list(range(epochs)), history['train_auc'], label = 'train auc')
plt.plot(list(range(epochs)), history['val_auc'], label = 'val auc')
plt.legend()<jupyter_output><empty_output><jupyter_text>make predictions<jupyter_code>predictor = training.Predictor(model_dir='trained_model')
pred_dict = predictor.predict(df_test.head())
pred_dict
df_test_ = df_test.drop('target', axis=1)
pred_dict_ = predictor.predict(df_test_.head())
pred_dict_<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14156521739130434, 'fraction_numerical': 0.1071304347826087, 'mean_word_length': 3.707037643207856, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '192672', 'license': 'FTL', 'n_tokens_mistral': 1337, 'n_tokens_neox': 1120, 'n_words': 240, 'path': 'notebooks/test_notebook.ipynb', 'repo_name': 'howewenann/fake_news', 'size': '2875'}