<jupyter_start><jupyter_text>Tokenize and encode<jupyter_code>model_ckpt = "roberta-large"
tokenizer = AutoTokenizer.from_pretrained(model_ckpt, problem_type="multi_label_classification")
# max_length cannot be set in our transformers-multilabel-text-classification-without-problem-type-and-datasets.ipynb
def tokenize_and_encode(examples):
    return tokenizer.batch_encode_plus(examples["excerpt"], truncation=True, padding=True, max_length=120)
cols = ds["train"].column_names
cols.remove("labels")
ds_enc = ds.map(tokenize_and_encode, batched=True, remove_columns=cols, num_proc=2)
ds_enc
# cast label IDs to floats
ds_enc.set_format("torch")
ds_enc = (ds_enc
          .map(lambda x : {"float_labels": x["labels"].to(torch.float)}, remove_columns=["labels"])
          .rename_column("float_labels", "labels"))<jupyter_output><empty_output><jupyter_text>Load model<jupyter_code>num_labels=5
# Setting output_hidden_states to True will cause the epoch evaluation to fail, because the output will be the last layer embeddings and not the loss
# Enable this feature only during predictions
# output_hidden_states=True
model = AutoModelForSequenceClassification.from_pretrained('../data/mlm/model', num_labels=num_labels, problem_type="multi_label_classification").to('cuda')<jupyter_output>Some weights of the model checkpoint at ../data/mlm/model were not used when initializing RobertaForSequenceClassification: ['lm_head.decoder.bias', 'lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'lm_head.dense.weight', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.layer_norm.weight']
- This IS expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing RobertaForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at ../data/mlm/model and are newly initialized: ['classifier.dense.weight', 'classifier.out_proj.bias', 'classif[...]<jupyter_text>Load trainer<jupyter_code>ds_enc["train"][0]
batch_size = 16
# configure logging so we see training loss
logging_steps = len(ds_enc['train']) // batch_size

training_args = TrainingArguments(
    output_dir='../data/seqclas/results0',                  # output directory
    evaluation_strategy = "epoch",                          # evaluate model after every epoch
    num_train_epochs=15,                                    # total number of training epochs
    learning_rate=1e-5,                                     # starting learning rate
    per_device_train_batch_size=batch_size,                 # batch size per device during training
    per_device_eval_batch_size=batch_size,                  # batch size for evaluation
    warmup_steps=500,                                       # number of warmup steps for learning rate scheduler
    weight_decay=0.01,                                      # strength of weight decay
    logging_dir='../data/seqclas/logs0',                    # directory for storing logs
    logging_steps=logging_steps,
    report_to="none",
    seed=seed,
    fp16=True,
    eval_accumulation_steps=5,
)
def compute_metrics(eval_pred):
    logits, labels = eval_pred
    predictions = np.argmax(logits, axis=-1)
    converted_labels = [int(i) for i in lb.inverse_transform(labels)]
    return {'f1_score': f1_score(y_true=converted_labels, y_pred=predictions, average='weighted'), 
            'accuracy': accuracy_score(y_true=converted_labels, y_pred=predictions)}
trainer = Trainer(
    model = model,
    args = training_args,
    train_dataset = ds_enc['train'],
    eval_dataset = ds_enc['valid'],
    compute_metrics = compute_metrics,
    tokenizer = tokenizer
)
trainer.evaluate()
trainer.train()
trainer.evaluate()
trainer.predict(test_dataset=ds_enc['test'])[2]
model.save_pretrained('../data/seqclas/model0')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.0943531093638313, 'fraction_numerical': 0.006671431975220395, 'mean_word_length': 4.254067584480601, 'pattern_counts': {'":': 1, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '526246', 'license': 'Apache-2.0', 'n_tokens_mistral': 1243, 'n_tokens_neox': 1202, 'n_words': 295, 'path': 'notebook/models/transformers_multilabel_text_classification_with_fine_tuned_weights_problem_type_and_datasets.ipynb', 'repo_name': 'vinay9986/CommonLit', 'size': '4197'}