<jupyter_start><jupyter_text>Model<jupyter_code>indexer = PretrainedTransformerMismatchedIndexer(model_name=ROBERTA_MODEL)
wordpieces, offsets = indexer._allennlp_tokenizer.intra_word_tokenize(['We', 'are', 'living', 'in', 'Mönchengladbach'])
wordpieces
reader = SequenceTaggingDatasetReader(token_indexers={'tokens' : indexer})
train_dataset = reader.read('data/train.seq')
dev_dataset = reader.read('data/dev.seq')
test_dataset = reader.read('data/trial.seq')
vocab = Vocabulary.from_instances(train_dataset.instances)
train_dataset.index_with(vocab)
dev_dataset.index_with(vocab)
test_dataset.index_with(vocab)
from allennlp.modules.token_embedders import PretrainedTransformerMismatchedEmbedder
from allennlp.modules.text_field_embedders import BasicTextFieldEmbedder
from allennlp.modules.seq2seq_encoders import PassThroughEncoder


embedder = PretrainedTransformerMismatchedEmbedder(model_name=ROBERTA_MODEL)
text_field_embedder = BasicTextFieldEmbedder({'tokens': embedder})
seq2seq_encoder = PassThroughEncoder(input_dim=embedder.get_output_dim())
parameter_name_overrides, positions = get_parameter_overrides()
pretrained_roberta = PretrainedModelInitializer(pretrained_roberta_bin, 
                                                parameter_name_overrides)
initializer = InitializerApplicator([(positions, pretrained_roberta)])
from allennlp.models import SimpleTagger


model = SimpleTagger(text_field_embedder=text_field_embedder, 
                      vocab=vocab, 
                      encoder=seq2seq_encoder,
                      calculate_span_f1=True,
                      label_encoding='IOB1',
                     initializer=initializer).cuda(device=cuda_device)
import torch.optim as optim
from transformers import AdamW
import transformers
from datetime import datetime

from allennlp.training.learning_rate_schedulers import LinearWithWarmup
from torch.utils.data import DataLoader
from allennlp.training import GradientDescentTrainer
from allennlp.training.learning_rate_schedulers import SlantedTriangular

import math


num_epochs = 10
batch_size = 10
steps_per_epoch = math.ceil(len(train_dataset) / batch_size)

optimizer = optim.AdamW(model.parameters(), lr=3e-5, weight_decay=0.01)
train_data_loader = DataLoader(dataset=train_dataset, batch_size=batch_size, 
                               collate_fn=allennlp_collate, shuffle=True)
val_data_loader = DataLoader(dataset=dev_dataset, batch_size=50, collate_fn=allennlp_collate)
lr_scheduler = LinearWithWarmup(optimizer, 
                                num_epochs=num_epochs, 
                                warmup_steps=(steps_per_epoch*num_epochs)*0.1, 
                                num_steps_per_epoch=steps_per_epoch)

date_time = datetime.now()
date_str = date_time.strftime('%m/%d/%Y')
time_str = date_time.strftime('%H:%M:%S')


trainer = GradientDescentTrainer(
    model=model,
    optimizer=optimizer,
    data_loader=train_data_loader,
    validation_data_loader=val_data_loader,
    num_epochs=num_epochs,
    cuda_device=cuda_device,
    learning_rate_scheduler=lr_scheduler,
    num_gradient_accumulation_steps=1,
    serialization_dir=f'./workdir/{date_str}/{time_str}',
    grad_clipping=1.
)

try:
    metrics = trainer.train()
except KeyboardInterrupt:
    pass
torch.save(model.state_dict(), '../toxicity/models/roberta_spans')
test_dataloader = DataLoader(test_dataset, batch_size=100, collate_fn=allennlp_collate)
evaluate(model, test_dataloader, cuda_device=cuda_device)
predictor = SentenceTaggerPredictor(model, reader)
preds = predictor.predict('We are living in New York')
print(list(zip(preds['words'], preds['tags'])))<jupyter_output>[('We', 'O'), ('are', 'O'), ('living', 'O'), ('in', 'O'), ('New', 'O'), ('York', 'O')]<jupyter_text>Predicting  in conll format<jupyter_code>from nltk import word_tokenize
def conll_sentence(labeled_sentence):
    ok_sentence = []
    for elem in labeled_sentence:
        new_word = []
        word, label = elem
        word_elems = word_tokenize(word)
        for word_elem in word_elems:
            if any(char.isalpha() for char in word_elem):
                new_word.append((word_elem, label))
            else:
                new_word.append((word_elem, 'O'))
        ok_sentence += new_word
    return ok_sentence
import pandas as pd
from ast import literal_eval
trial = pd.read_csv('data/tsd_trial.csv')
trial['spans'] = trial.spans.apply(literal_eval)
conll_preds = []
for text in trial.text:
    preds = predictor.predict(text)
    raw_pred = list(zip(preds['words'], preds['tags']))
    conll_pred = conll_sentence(raw_pred)
    conll_preds.append(conll_pred)
with open('./data/preds_trial.conll2003', 'w') as f:
    for sentence in conll_preds:
        for comb in sentence:
            f.write(comb[0] + ' ' + comb[1] + '\n')
        f.write('\n')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10272277227722772, 'fraction_numerical': 0.005981848184818482, 'mean_word_length': 4.423937360178971, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '647531', 'license': 'Apache-2.0', 'n_tokens_mistral': 1641, 'n_tokens_neox': 1583, 'n_words': 256, 'path': 'semeval2021/allennlp_from_pretrained.ipynb', 'repo_name': 'skoltech-nlp/toxic-span-detection', 'size': '4848'}