<jupyter_start><jupyter_text>[jump](JUMP)<jupyter_code>tokenizer = BertTokenizer.from_pretrained(NORBERT)
bert = BertModel.from_pretrained(NORBERT)
bert_ftc = BertForTokenClassification.from_pretrained(
    NORBERT,
    num_labels=len(train_dataset.label_vocab),
    output_attentions = False,
    output_hidden_states = False
)
X, y = train_dataset[0]
# print(tokenizer(X)['input_ids'])
y
X
from packages.preprocess import NewCONLLUDataset
new_train_dataset = NewCONLLUDataset(
    con_df,
    tokenizer,
)
new_valid_dataset = NewCONLLUDataset(
    con_df,
    tokenizer,
    new_train_dataset.label_vocab
)
new_train_dataset[0]
for i in range (4):
    print(len(new_train_dataset[i][0]))
def pad(batch): # pass padding index
    longest_sentence = max([X.size(0) for X, y, z in batch])

    new_X, new_y, new_z = [], [], []

    for X, y, z in batch:
        new_X.append(torch.nn.functional.pad(
            X,
            (0, longest_sentence - X.size(0)),
            value=0)
        )
        new_y.append(torch.nn.functional.pad(
            y,
            (0, longest_sentence - y.size(0)),
            value=0)
        )
        new_z.append(torch.nn.functional.pad(
            z,
            (0, longest_sentence - z.size(0)),
            value=0)
        )


    new_X = torch.stack(new_X).long()
    new_y = torch.stack(new_y).long()
    new_z = torch.stack(new_z).long()

    return new_X, new_y, new_z
# creating data loaders
new_train_loader = DataLoader(new_train_dataset, batch_size=32, 
collate_fn=pad)
new_val_loader = DataLoader(new_valid_dataset, batch_size=32, 
collate_fn=pad)
for batch in new_train_loader:
    X, y, z = batch
    # print('X:', X)
    # print('y:', y)
    # print('z:', z)
    
    print(bert_ftc(X, labels=y, attention_mask=z).loss.item())


len(x_tkn[1])
tokenizer()
# train_dataset[9][1]
# label_vocab_len = 18

# def label_idx_to_tensor(label):
#     lbl_split = label
#     r = torch.zeros((len(lbl_split), label_vocab_len))
#     for i, ele in enumerate(lbl_split):
#         r[i][int(ele)] = 1
#     return r

# for i in range(4):
#     print(label_idx_to_tensor(train_dataset[i][1]))<jupyter_output><empty_output><jupyter_text>JUMP<jupyter_code># calling transformer model
print('Initializing model')
transformer = Transformer(
    NORBERT=NORBERT,
    num_labels=len(train_dataset.label_vocab),
    device='cpu',
    d_model=768,
    nhead=8,
    num_encoder_layers=6,
    num_decoder_layers=6,
    dim_feedforward=2048,
    dropout=0.2,
    activation='relu',
    epochs=10,
    lr_scheduler=False,
    factor=0.1,
    patience=2,
    loss_funct='binary-cross-entropy',
    random_state=1,
    verbose=True
)

print('Fitting model')
transformer.fit(
    loader=train_loader,
    verbose=True
)

from packages.model import BERT
# calling transformer model
print('Initializing model')
bert = BERT(
    NORBERT=NORBERT,
    num_labels=len(train_dataset.label_vocab), #TODO automize
    device='cpu',
    epochs=5, # tune
    pad_idx = train_dataset.label_indexer['O'],
    random_state=1,
    verbose=True
)


print('Fitting model')
bert.fit(
    loader=train_loader,
    verbose=True
)<jupyter_output>Initializing model
Some weights of the model checkpoint at /cluster/shared/nlpl/data/vectors/latest/216/ were not used when initializing BertForTokenClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.predictions.decoder.bias', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias']
- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some weights of BertForTo[...]
{'fraction_non_alphanumeric': 0.11546788549434689, 'fraction_numerical': 0.011787346644214578, 'mean_word_length': 3.9558998808104886, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '639307', 'license': 'MIT', 'n_tokens_mistral': 1480, 'n_tokens_neox': 1431, 'n_words': 288, 'path': 'Oblig3/norbert_.ipynb', 'repo_name': 'fabiorodp/IN5550_Neural_Methods_in_Natural_Language_Processing', 'size': '4157'}