<jupyter_start><jupyter_text>[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/Mustapha-AJEGHRIR/medical_txt_parser/blob/main/src/notebooks/assertions_nli/ast_nli_scibert.ipynb)  Relations classificationBased of: https://github.com/huggingface/transformers/blob/master/examples/pytorch/text-classification/run_xnli.py<jupyter_code>!nvidia-smi
%%capture
!pip install seqeval transformers datasets spacy sentence_transformers
from google.colab import drive
drive.mount('/content/drive')

%cd /content/drive/MyDrive/projects/medical_txt_parser
%reload_ext autoreload
%autoreload 2

import warnings
warnings.filterwarnings("ignore")

path = %pwd
while "src" in path:
    %cd ..
    path = %pwd

import glob
import pandas as pd
import os
import numpy as np
from tqdm import tqdm
from pprint import pprint
import matplotlib.pyplot as plt
import random

import logging
import os
import random
import sys
from dataclasses import dataclass, field
from typing import Optional

from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report

import datasets
import numpy as np
from datasets import load_dataset, load_metric , Dataset
import transformers
from transformers import (
    AutoConfig,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    DataCollatorWithPadding,
    EvalPrediction,
    HfArgumentParser,
    Trainer,
    TrainingArguments,
    default_data_collator,
    set_seed,
)
from transformers.trainer_utils import get_last_checkpoint
from transformers.utils import check_min_version
from transformers.utils.versions import require_version
from transformers import pipeline

require_version("datasets>=1.8.0", "To fix: pip install --upgrade datasets")

from src.utils.parse_data import parse_ast, parse_concept, parse_relation
train_data_path = "data/train"
val_data_path = "data/val"
ast_folder_name = "ast"
concept_folder_name = "concept"
rel_folder_name = "rel"
txt_folder_name = "txt"
nli_data_path = "data/nli"
re_data_path = "data/re"

# model args
model_name_or_path = "allenai/scibert_scivocab_uncased" # "gsarti/scibert-nli"
cache_dir = None
model_revision = None 
tokenizer_name = model_name_or_path
do_lower_case = None
use_fast_tokenizer = True
fp16 = True

# data args
pad_to_max_length = None
max_seq_length = None

set_seed(42)<jupyter_output><empty_output><jupyter_text>Import data<jupyter_code>relations_df = pd.read_csv(re_data_path + os.sep + "re_scibert_data.tsv", sep="\t", header=None)
relations_df.columns = ["text", "label"]
label2id = {label: i for i, label in enumerate(relations_df["label"].value_counts().index.tolist())}
id2label = {i: label for label, i in label2id.items()}
relations_df["label"] = relations_df.label.map(label2id)
relations_df
# Build HuggingFace Dataset

train_df, val_df = train_test_split(relations_df, train_size=None, shuffle=True, test_size=0.2, stratify=relations_df["label"], random_state=42)

features = datasets.Features({'text': datasets.Value(dtype='string'),
 'label': datasets.ClassLabel(num_classes=len(id2label), names=list(id2label.values()))})

train_dataset = Dataset.from_pandas(train_df, preserve_index=False, features=features)
eval_dataset = Dataset.from_pandas(val_df, preserve_index=False, features=features)

label_list = train_dataset.features["label"].names
num_labels = len(label_list)

label_list = train_dataset.features["label"].names
train_dataset, eval_dataset
# check labels balance
print(f"train_df labels: {train_df['label'].value_counts()}")
print(f"val_df labels: {val_df['label'].value_counts()}")
# Load pretrained model and tokenizer
# In distributed training, the .from_pretrained methods guarantee that only one local process can concurrently
# download model & vocab.
config = AutoConfig.from_pretrained(
    model_name_or_path,
    num_labels=num_labels,
    finetuning_task="re",
    cache_dir=cache_dir,
    revision=model_revision,
    label2id=label2id,
    id2label=id2label
)
tokenizer = AutoTokenizer.from_pretrained(
    tokenizer_name if tokenizer_name else model_name_or_path,
    # do_lower_case=do_lower_case,
    cache_dir=cache_dir,
    use_fast=use_fast_tokenizer,
    revision=model_revision,
)
model = AutoModelForSequenceClassification.from_pretrained(
    model_name_or_path,
    from_tf=bool(".ckpt" in model_name_or_path),
    config=config,
    cache_dir=cache_dir,
    revision=model_revision,
)
# Preprocessing the datasets
# Padding strategy
if pad_to_max_length:
    padding = "max_length"
else:
    # We will pad later, dynamically at batch creation, to the max sequence length in each batch
    padding = False


def preprocess_function(examples):
    # Tokenize the texts
    return tokenizer(
        examples["text"],
        padding=padding,
        max_length=max_seq_length,
        truncation=True,
    )


train_dataset = train_dataset.map(
    preprocess_function,
    batched=True,
    desc="Running tokenizer on train dataset",
)
eval_dataset = eval_dataset.map(
    preprocess_function,
    batched=True,
    desc="Running tokenizer on validation dataset",
)

# Log a few random samples from the training set:
for index in random.sample(range(len(train_dataset)), 3):
    print(f"Sample {index} of the training set: {train_dataset[index]}.\n")

# Get the metric function
f1_metric = load_metric("f1")
precision_metric = load_metric("precision")
recall_metric = load_metric("recall")
accuracy_metric = load_metric("accuracy")

# You can define your custom compute_metrics function. It takes an `EvalPrediction` object (a namedtuple with a
# predictions and label_ids field) and has to return a dictionary string to float.
def compute_metrics(p: EvalPrediction):
    preds = p.predictions[0] if isinstance(p.predictions, tuple) else p.predictions
    preds = np.argmax(preds, axis=1)

    metrics = {}
    metrics.update(f1_metric.compute(predictions=preds, references=p.label_ids, average="macro"))
    metrics.update(precision_metric.compute(predictions=preds, references=p.label_ids, average="macro"))
    metrics.update(recall_metric.compute(predictions=preds, references=p.label_ids, average="macro"))
    metrics.update(accuracy_metric.compute(predictions=preds, references=p.label_ids))
    return metrics
# Data collator will default to DataCollatorWithPadding, so we change it if we already did the padding.
if pad_to_max_length:
    data_collator = default_data_collator
elif fp16:
    data_collator = DataCollatorWithPadding(tokenizer, pad_to_multiple_of=8)
else:
    data_collator = None
model_name_or_path
# address class imbalance 
import torch
from torch import nn
from transformers import Trainer

class_weights = [len(train_df)/ (len(train_df[train_df["label"] == i])*len(id2label)) for i in id2label.keys()]
class_weights = torch.tensor(class_weights).log1p()

class CustomTrainer(Trainer):
    def compute_loss(self, model, inputs, return_outputs=False):
        labels = inputs.get("labels")
        # forward pass
        outputs = model(**inputs)
        logits = outputs.get("logits")
        loss_fct = nn.CrossEntropyLoss(weight=torch.tensor(class_weights).to(logits.device))
        loss = loss_fct(logits.view(-1, self.model.config.num_labels), labels.view(-1))
        return (loss, outputs) if return_outputs else loss
# Initialize our Trainer
model_folder_name = f"{model_name_or_path.split('/')[-1]}-re-1"
args = TrainingArguments(
    f"training_logs/{model_folder_name}",
    evaluation_strategy = "epoch",
    save_strategy = "no",
    learning_rate=1e-5,
    per_device_train_batch_size=32,
    per_device_eval_batch_size=32,
    num_train_epochs=10,
    weight_decay=0.05,
    logging_steps=1,
    warmup_ratio=0.1,
)

trainer = CustomTrainer(
    model=model,
    args=args,
    train_dataset=train_dataset ,
    eval_dataset=eval_dataset ,
    compute_metrics=compute_metrics,
    tokenizer=tokenizer,
    data_collator=data_collator,
)
train_result = trainer.train()
metrics = train_result.metrics
metrics["train_samples"] = len(train_dataset)

trainer.save_model(f"models/{model_folder_name}")  # Saves the tokenizer too for easy upload

trainer.log_metrics("train", metrics)
trainer.save_metrics("train", metrics)
print("*** Evaluate ***") 
metrics = trainer.evaluate(eval_dataset=eval_dataset)

metrics["eval_samples"] = len(eval_dataset)

trainer.log_metrics("eval", metrics)
trainer.save_metrics("eval", metrics)
predictions, labels, _ = trainer.predict(eval_dataset, metric_key_prefix="predict")
predictions = np.argmax(predictions, axis=1)
print(classification_report(labels, predictions,target_names=label_list))
predictions, labels, _ = trainer.predict(train_dataset, metric_key_prefix="predict")
predictions = np.argmax(predictions, axis=1)
print(classification_report(labels, predictions,target_names=label_list))<jupyter_output>The following columns in the test set  don't have a corresponding argument in `BertForSequenceClassification.forward` and have been ignored: text.
***** Running Prediction *****
  Num examples = 12274
  Batch size = 64<jupyter_text>Evaluate the model<jupyter_code>model_folder_name
!ls models
# Local model
# label_list = ['TeCP', 'TrIP', 'TrNAP', 'TrAP', 'TrCP', 'TrWP', 'TeRP', 'PIP']

# id2label = {i: label for i, label in enumerate(label_list)}
# label2id = {v: k for k, v in id2label.items()}

model_folder_name = f"{model_name_or_path.split('/')[-1]}-re-1"
model_checkpoint = f"models/{model_folder_name}"
model = AutoModelForSequenceClassification.from_pretrained(f"models/{model_folder_name}", label2id=label2id, id2label=id2label)
tokenizer = AutoTokenizer.from_pretrained(f"models/{model_folder_name}")

# Data collator will default to DataCollatorWithPadding, so we change it if we already did the padding.
if pad_to_max_length:
    data_collator = default_data_collator
elif fp16:
    data_collator = DataCollatorWithPadding(tokenizer, pad_to_multiple_of=8)
else:
    data_collator = None
    

trainer = Trainer(
    model=model,
    # args=args,
    # train_dataset=train_dataset ,
    # eval_dataset=eval_dataset ,
    # compute_metrics=compute_metrics,
    tokenizer=tokenizer,
    data_collator=data_collator,
)
val_data_path + os.sep + txt_folder_name + os.sep +  "*.txt"
val_data_path + os.sep + concept_folder_name + os.sep + fname + ".con"
test_data_path = "data/test"
text_files = glob.glob(test_data_path + os.sep + txt_folder_name + os.sep +  "*.txt")
filename = ""
df = pd.DataFrame()
for file in tqdm(text_files):
    with open(file, 'r') as f:
        text = f.read()
        # split lines
        lines = text.split('\n')
        filename =[ file.split("/")[-1].split(".")[0]] * len(lines)
        df = df.append(pd.DataFrame({"text": lines, "filename": filename, "line_number": range(len(lines))}), ignore_index=True)

df = df.sort_values(by=["filename", "line_number"])
# remove empty text lines
# df = df[df.text != ""]
# # reset index
# df = df.reset_index(drop=True)

# add concepts
rel_df = pd.DataFrame()
for fname in tqdm(df["filename"].unique()):
    concept_dict = parse_concept(test_data_path + os.sep + concept_folder_name + os.sep + fname + ".con")
    
    concept_df = pd.DataFrame(concept_dict).drop(columns=["end_line"])
    test_concept_df = concept_df[concept_df["concept_type"] == "test"]
    problem_concept_df = concept_df[concept_df["concept_type"] == "problem"]
    treatment_concept_df = concept_df[concept_df["concept_type"] == "treatment"]

    # class test --> problem
    test_problem_df = pd.merge(test_concept_df, problem_concept_df, how="inner", on="start_line")

    # class treatment --> problem
    treatment_problem_df = pd.merge(treatment_concept_df, problem_concept_df, how="inner", on="start_line")

    # class problem --> problem
    problem_problem_df = pd.merge(problem_concept_df, problem_concept_df, how="inner", on="start_line")
    problem_problem_df = problem_problem_df[problem_problem_df["concept_text_x"] != problem_problem_df["concept_text_y"]] # TODO: remove duplicates ?

    tmp = pd.concat([test_problem_df, treatment_problem_df, problem_problem_df], axis=0)
    tmp["filename"] = fname
    rel_df = rel_df.append(tmp, ignore_index=True)
            
rel_df = rel_df.sort_values(by=["filename", "start_line"])
rel_df = rel_df.reset_index(drop=True)
rel_df
rel_df = rel_df[[ "filename", "start_line", "concept_text_x", "concept_text_y", "concept_type_x", "concept_type_y", "start_word_number_x", "end_word_number_x", "start_word_number_y", "end_word_number_y"]]
rel_df
# make predict dataset
def preprocess_text(row):
    # find line
    line =  df[(df["filename"] == row["filename"]) & (df["line_number"] == row["start_line"]-1)]["text"].values[0]
    # line = line.lower()
    line = " ".join(line.split()) # remove multiple spaces

    concept_text_x = "<< "+ " ".join(line.split()[row["start_word_number_x"]:row["end_word_number_x"]+1]) + " >>"
    concept_text_y = "[[ " + " ".join(line.split()[row["start_word_number_y"]:row["end_word_number_y"]+1]) + " ]]"
    start_word_number_x = row["start_word_number_x"]
    end_word_number_x = row["end_word_number_x"]
    start_word_number_y = row["start_word_number_y"]
    end_word_number_y = row["end_word_number_y"]

    if row["start_word_number_x"] > row["start_word_number_y"]:
        concept_text_x, concept_text_y = concept_text_y, concept_text_x
        start_word_number_x, start_word_number_y = start_word_number_y, start_word_number_x
        end_word_number_x, end_word_number_y = end_word_number_y, end_word_number_x
    text = " ".join(line.split()[: start_word_number_x] + [concept_text_x] + line.split()[end_word_number_x+1: start_word_number_y] + [concept_text_y] + line.split()[end_word_number_y+1:])

    row["text"] = text
    return row

predict_df = rel_df.apply(preprocess_text, axis=1)
predict_df
predict_dataset = Dataset.from_pandas(predict_df, preserve_index=False)
# predict_dataset = predict_dataset.select(range(10))
predict_dataset = predict_dataset.map(
                preprocess_function,
                batched=True,
                desc="Running tokenizer on prediction dataset",
            )
predict_dataset
 predictions, labels, metrics = trainer.predict(predict_dataset, metric_key_prefix="predict")
 predictions = np.argmax(predictions, axis=1)
 len(predictions)
rel_df["prediction"] = [id2label[label] for label in predictions]
rel_df
# for each file create <filename>.con
os.makedirs(test_data_path + os.sep + rel_folder_name, exist_ok=True)
# empty folder if exists
files = glob.glob(test_data_path + os.sep + rel_folder_name + os.sep + "*.rel")
for file in files:
    os.remove(file)

for i, row in tqdm(rel_df.iterrows()):
    filename = row["filename"]
    concept_text_x = row["concept_text_x"]
    concept_text_y = row["concept_text_y"]
    concept_type_x = row["concept_type_x"]
    concept_type_y = row["concept_type_y"]
    start_word_number_x = row["start_word_number_x"]
    end_word_number_x = row["end_word_number_x"]
    start_word_number_y = row["start_word_number_y"]
    end_word_number_y = row["end_word_number_y"]
    line_number = row["start_line"]
    prediction = row["prediction"]
    if prediction != "Other":
        with open(test_data_path + os.sep + rel_folder_name + os.sep + filename + ".rel", "a") as f:
            # fill like this c="pefusion imaging" 19:6 19:7||r="TeRP"||c="perfusion defects" 19:12 19:13
            f.write(
                f"c=\"{concept_text_x}\" {line_number}:{start_word_number_x} {line_number}:{end_word_number_x}||r=\"{prediction}\"||c=\"{concept_text_y}\" {line_number}:{start_word_number_y} {line_number}:{end_word_number_y}\n"
            )
    

rel_files = glob.glob(test_data_path + os.sep + rel_folder_name + os.sep + "*.rel")
rel_files = [f.split(os.sep)[-1][:-4] for f in rel_files]
txt_files = [f.split(os.sep)[-1][:-4] for f in text_files]
# find missing files
missing_files = set(txt_files) - set(rel_files)
missing_files

# create empty files for missing files
for f in missing_files:
    with open(test_data_path + os.sep + rel_folder_name + os.sep + f + ".rel", "w") as f:
        f.write("")
rel_files = glob.glob(test_data_path + os.sep + rel_folder_name + os.sep + "*.rel")
rel_files = [f.split(os.sep)[-1][:-4] for f in rel_files]
txt_files = [f.split(os.sep)[-1][:-4] for f in text_files]
# find missing files
missing_files = set(txt_files) - set(rel_files)
missing_files
!cat data/val/rel/0006.rel
!zip -r scibert-test-rel-2.zip data/test/rel/<jupyter_output>huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...
To disable this warning, you can either:
	- Avoid using `tokenizers` before the fork if possible
	- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)
  adding: data/test/rel/ (stored 0%)
  adding: data/test/rel/0233.rel (deflated 69%)
  adding: data/test/rel/0463.rel (deflated 74%)
  adding: data/test/rel/0427.rel (deflated 79%)
  adding: data/test/rel/0058.rel (deflated 77%)
  adding: data/test/rel/0460.rel (deflated 78%)
  adding: data/test/rel/0265.rel (stored 0%)
  adding: data/test/rel/0214.rel (deflated 70%)
  adding: data/test/rel/0021.rel (deflated 65%)
  adding: data/test/rel/0415.rel (deflated 53%)
  adding: data/test/rel/0005.rel (deflated 72%)
  adding: data/test/rel/0281.rel (deflated 76%)
  adding: data/test/rel/0037.rel (deflated 75%)
  adding: data/test/rel/0129.rel (deflated 71%)
  adding: data/tes[...]
{'fraction_non_alphanumeric': 0.12021075539774355, 'fraction_numerical': 0.010995933795315275, 'mean_word_length': 5.249821045096636, 'pattern_counts': {'":': 4, '<': 14, '<?xml version=': 0, '>': 19, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '460041', 'license': 'Apache-2.0', 'n_tokens_mistral': 6095, 'n_tokens_neox': 5901, 'n_words': 1203, 'path': 'src/notebooks/relations_re/re_clf_scibert.ipynb', 'repo_name': 'techthiyanes/medical_txt_parser', 'size': '17461'}