<jupyter_start><jupyter_text>Training<jupyter_code>model_name = model_checkpoint.split("/")[-1]
args = TrainingArguments(
    f"training_logs/{model_name}-finetuned-{task}",
    evaluation_strategy = "epoch",
    # learning_rate=1e-5,
    learning_rate=1e-5,
    per_device_train_batch_size=batch_size,
    per_device_eval_batch_size=batch_size,
    num_train_epochs=20,
    weight_decay=0.05,
    logging_steps=5,

    warmup_ratio=0.1,
)
data_collator = DataCollatorForTokenClassification(tokenizer)


metric = load_metric("seqeval")
def compute_metrics(p):
    predictions, labels = p
    predictions = np.argmax(predictions, axis=2)

    # Remove ignored index (special tokens)
    true_predictions = [
        [label_list[p] for (p, l) in zip(prediction, label) if l != -100]
        for prediction, label in zip(predictions, labels)
    ]
    true_labels = [
        [label_list[l] for (p, l) in zip(prediction, label) if l != -100]
        for prediction, label in zip(predictions, labels)
    ]

    results = metric.compute(predictions=true_predictions, references=true_labels)
    return {
        "precision": results["overall_precision"],
        "recall": results["overall_recall"],
        "f1": results["overall_f1"],
        "accuracy": results["overall_accuracy"],
    }
trainer = Trainer(
    model,
    args,
    train_dataset=D["train"],
    eval_dataset=D["val"],
    data_collator=data_collator,
    tokenizer=tokenizer,
    compute_metrics=compute_metrics, 
)
trainer.train()
# trainer.model = AutoModelForTokenClassification.from_pretrained("./training_logs/electramed_base_scivocab_1M-finetuned-ner/checkpoint-2500", local_files_only=True)
# trainer.model.to("cuda")<jupyter_output><empty_output><jupyter_text>Performence on validation<jupyter_code>predictions, labels, _ = trainer.predict(D["val"])
predictions = np.argmax(predictions, axis=2)

# Remove ignored index (special tokens)
true_predictions = [
    [label_list[p] for (p, l) in zip(prediction, label) if l != -100]
    for prediction, label in zip(predictions, labels)
]
true_labels = [
    [label_list[l] for (p, l) in zip(prediction, label) if l != -100]
    for prediction, label in zip(predictions, labels)
]

results = metric.compute(predictions=true_predictions, references=true_labels)
results
trainer.save_model("logs/scibert_20_epochs_64_batch_99_train_split")<jupyter_output>Saving model checkpoint to logs/scibert_20_epochs_64_batch_99_train_split
Configuration saved in logs/scibert_20_epochs_64_batch_99_train_split/config.json
Model weights saved in logs/scibert_20_epochs_64_batch_99_train_split/pytorch_model.bin
tokenizer config file saved in logs/scibert_20_epochs_64_batch_99_train_split/tokenizer_config.json
Special tokens file saved in logs/scibert_20_epochs_64_batch_99_train_split/special_tokens_map.json<jupyter_text>Reformat to the initial format<jupyter_code># prediction = np.load("../data/prediction.npy")
# print("min :", min(prediction.reshape(-1)))
# print("max :", max(prediction.reshape(-1)))
# prediction = predictions_2
test_data = D["test"]
prediction, _, _ = trainer.predict(D["test"])
prediction = np.argmax(prediction, axis=2)
label_list
token_to_label = {token: token.split("-")[-1] for token in label_list}
token_id_to_label = {i: token_to_label[token].lower() for i, token in enumerate(label_list)}
token_id_to_label
max(predictions.reshape(-1))
ast_to_concept = {
    "test" : "test",
    "treatment" : "treatment",
    "present" : "problem",
    "absent" : "problem",
    "possible" : "problem",
    "conditional" : "problem",
    "hypothetical" : "problem",
    "associated_with_someone_else" : "problem"
}
from utils.save_predictions import save_predictions

# save_predictions(test_data, prediction)
save_predictions(test_data, prediction)<jupyter_output><empty_output><jupyter_text>Viz<jupyter_code>effect_ner_model = pipeline(task="ner", model=model, tokenizer=tokenizer, device=0)
effect_ner_model(D["train"][4]["text"])
from spacy import displacy
def visualize_entities(sentence):
    tokens = effect_ner_model(sentence)
    entities = []

    for token in tokens:
        label = int(token["entity"][-1])
        if label != 0:
            token["label"] = label_list[label]
            entities.append(token)

    params = [{"text": sentence, "ents": entities, "title": None}]

    html = displacy.render(
        params,
        style="ent",
        manual=True,
        jupyter=True,
        options={
            "colors": {
                "B-PROBLEM": "#f08080",
                "I-PROBLEM": "#f08080",
                "B-TEST": "#9bddff",
                "I-TEST": "#9bddff",
                "B-TREATMENT": "#ffdab9",
                "I-TREATMENT": "#ffdab9",
            },
        },
    )

# pick 5 random sentences from the test set
for i in range(5):
    index = np.random.randint(0, len(D["train"]))
    visualize_entities(D["train"][index]["text"])
    print(f"Text: {D['train'][index]['text']}")
    # print(f"Problems: {D['D'][index]['problem']}")
    print(f"Tests: {D['train'][index]['test']}")
    print(f"Treatments: {D['train'][index]['treatment']}")
    print(f"Predent: {D['train'][index]['present']}")
    print(f"{'*' * 50}\n")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14113366526235158, 'fraction_numerical': 0.018958253542703946, 'mean_word_length': 4.40124095139607, 'pattern_counts': {'":': 14, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '574400', 'license': 'Apache-2.0', 'n_tokens_mistral': 1867, 'n_tokens_neox': 1808, 'n_words': 311, 'path': 'src/notebooks/train_and_predict_con_ast.ipynb', 'repo_name': 'techthiyanes/medical_txt_parser', 'size': '5222'}