<jupyter_start><jupyter_text>Joint Modeling NotebookRun the following cells to train a joint classifier  0. Imports<jupyter_code>%load_ext autoreload
%autoreload 2
    
%load_ext tensorboard

import sys
sys.path.append('../jointclassifier/')
from joint_args import ModelArguments, DataTrainingArguments, TrainingArguments
from joint_dataloader import load_dataset
from joint_trainer import JointTrainer
from single_trainer import SingleTrainer
from joint_model_v1 import JointSeqClassifier

from transformers import HfArgumentParser, AutoConfig, AutoTokenizer
import os<jupyter_output><empty_output><jupyter_text>1. Initialize the Arguments<jupyter_code>task = "abstract+shakespeare"
data_dir = "../data/processed_filtered/"
model_name = "distilbert-base-uncased"
model_nick = "distilbert_uncased_2"
output_dir = "../models/"
freeze_encoder = "False"
skip_preclassifier = "False"
train_jointly = "True"
epochs = "3"
train_batch_size = "256"
eval_batch_size = "512"
log_save_steps = "200"

parser = HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments))
model_args, data_args, training_args = parser.parse_args_into_dataclasses([
    "--model_name_or_path",
    model_name,
    "--model_nick",
    model_nick,
    "--task",
    task,
    "--data_dir",
    data_dir,
    "--output_dir",
    os.path.join(output_dir, model_nick, task, 'joint'),
    "--cache_dir",
    os.path.join(output_dir,"cache"),
    "--freeze_encoder",
    freeze_encoder,
    "--skip_preclassifier",
    skip_preclassifier,
    "--train_jointly",
    train_jointly,
    "--overwrite_cache",
    "--per_device_train_batch_size",
    train_batch_size,
    "--per_device_eval_batch_size",
    eval_batch_size,
    "--max_seq_len",
    "64",
    "--gradient_accumulation_steps",
    "1",
    "--num_train_epochs",
    epochs,
    "--logging_steps",
    log_save_steps,
    "--save_steps",
    log_save_steps
])<jupyter_output>PyTorch: setting up devices
The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).<jupyter_text>2. Load the Tokenizer<jupyter_code>model_config = AutoConfig.from_pretrained(model_args.model_name_or_path, cache_dir=model_args.cache_dir)
tokenizer = AutoTokenizer.from_pretrained(model_args.model_name_or_path, cache_dir=model_args.cache_dir,
                                         model_max_length = data_args.max_seq_len)<jupyter_output>loading configuration file https://huggingface.co/distilbert-base-uncased/resolve/main/config.json from cache at ../models/cache/23454919702d26495337f3da04d1655c7ee010d5ec9d77bdb9e399e00302c0a1.d423bdf2f58dc8b77d5f5d18028d7ae4a72dcfd8f468e81fe979ada957a8c361
Model config DistilBertConfig {
  "activation": "gelu",
  "architectures": [
    "DistilBertForMaskedLM"
  ],
  "attention_dropout": 0.1,
  "dim": 768,
  "dropout": 0.1,
  "hidden_dim": 3072,
  "initializer_range": 0.02,
  "max_position_embeddings": 512,
  "model_type": "distilbert",
  "n_heads": 12,
  "n_layers": 6,
  "pad_token_id": 0,
  "qa_dropout": 0.1,
  "seq_classif_dropout": 0.2,
  "sinusoidal_pos_embds": false,
  "tie_weights_": true,
  "transformers_version": "4.4.0.dev0",
  "vocab_size": 30522
}

loading configuration file https://huggingface.co/distilbert-base-uncased/resolve/main/config.json from cache at ../models/cache/23454919702d26495337f3da04d1655c7ee010d5ec9d77bdb9e399e00302c0a1.d423bdf2f58dc8b77d5f5d18028d7ae4a7[...]<jupyter_text>3. Load the datasets Note : Single for Joint Training, Dict for Separate Training<jupyter_code>tasks = data_args.task.split('+')
train_dataset, idx_to_classes = load_dataset(data_args.data_dir, tokenizer, model_name=model_args.model_name_or_path, 
                            tasks=tasks, mode="train", n_proc=6000)
dev_dataset, _ = load_dataset(data_args.data_dir, tokenizer, model_name=model_args.model_name_or_path, 
                            tasks=tasks, mode="dev", n_proc=6000)<jupyter_output>100%|██████████| 14/14 [00:02<00:00,  6.23it/s]
 17%|█▋        | 1/6 [00:00<00:00,  7.59it/s]torch.Size([82119, 64]) torch.Size([82119, 64]) torch.Size([82119, 2]) torch.Size([82119])
100%|██████████| 6/6 [00:00<00:00,  6.94it/s]
  0%|          | 0/4 [00:00<?, ?it/s]torch.Size([116421, 64]) torch.Size([116421, 64]) torch.Size([116421, 2]) torch.Size([116421])
100%|██████████| 4/4 [00:00<00:00,  7.03it/s]
100%|██████████| 2/2 [00:00<00:00, 12.40it/s]torch.Size([20527, 64]) torch.Size([20527, 64]) torch.Size([20527, 2]) torch.Size([20527])
torch.Size([28286, 64]) torch.Size([28286, 64]) torch.Size([28286, 2]) torch.Size([28286])<jupyter_text>4. Initialize the Trainer and the Model & Train!<jupyter_code># # Open TensorBoard
# %tensorboard --logdir runs
label_dims = {task : 1 if len(list(idx_to_classes[task].keys())) == 2 else len(list(idx_to_classes[task].keys())) for task in idx_to_classes}
label_dims
print(f"Processing Joint Task : {tasks}")
model = JointSeqClassifier.from_pretrained(model_args.model_name_or_path,tasks=tasks, model_args=model_args,
                                                   task_if_single=None, joint = training_args.train_jointly,
                                                   label_dims=label_dims)
trainer = JointTrainer([training_args,model_args, data_args], model, train_dataset, dev_dataset, idx_to_classes)
trainer.train()<jupyter_output>Processing Joint Task : ['abstract', 'shakespeare']
loading configuration file https://huggingface.co/distilbert-base-uncased/resolve/main/config.json from cache at /home/vivek/.cache/huggingface/transformers/23454919702d26495337f3da04d1655c7ee010d5ec9d77bdb9e399e00302c0a1.d423bdf2f58dc8b77d5f5d18028d7ae4a72dcfd8f468e81fe979ada957a8c361
Model config DistilBertConfig {
  "activation": "gelu",
  "architectures": [
    "DistilBertForMaskedLM"
  ],
  "attention_dropout": 0.1,
  "dim": 768,
  "dropout": 0.1,
  "hidden_dim": 3072,
  "initializer_range": 0.02,
  "max_position_embeddings": 512,
  "model_type": "distilbert",
  "n_heads": 12,
  "n_layers": 6,
  "pad_token_id": 0,
  "qa_dropout": 0.1,
  "seq_classif_dropout": 0.2,
  "sinusoidal_pos_embds": false,
  "tie_weights_": true,
  "transformers_version": "4.4.0.dev0",
  "vocab_size": 30522
}

loading weights file https://huggingface.co/distilbert-base-uncased/resolve/main/pytorch_model.bin from cache at /home/vivek/.cache/huggingface/tran[...]<jupyter_text>5. Predict for a sentence<jupyter_code>model = JointSeqClassifier.from_pretrained(training_args.output_dir,tasks=tasks, model_args=model_args,
                                                   task_if_single=None, joint = training_args.train_jointly,
                                                label_dims=label_dims)
trainer = JointTrainer([training_args,model_args, data_args], model, train_dataset, dev_dataset, idx_to_classes)
sentence = "Thou shalt open the door!"
trainer.predict_for_sentence(sentence, tokenizer)
trainer.predict_for_sentence(sentence, tokenizer, salience=True)
trainer.evaluate()
for i in tokenizer(sentence).input_ids:
    print(tokenizer.decode(i))
tokenizer(sentence).input_ids<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1285240464344942, 'fraction_numerical': 0.07200110558319514, 'mean_word_length': 4.693941778127459, 'pattern_counts': {'":': 36, '<': 27, '<?xml version=': 0, '>': 21, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '660420', 'license': 'MIT', 'n_tokens_mistral': 2960, 'n_tokens_neox': 2632, 'n_words': 449, 'path': 'ml/notebooks/Joint Modeling.ipynb', 'repo_name': 'DavidThe4sian/marvin', 'size': '7236'}