<jupyter_start><jupyter_text>All imports here<jupyter_code>import dataclasses
import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Dict, Optional

import numpy as np
import pandas as pd
import json

from transformers import AutoConfig, AutoModelForSequenceClassification, AutoTokenizer, EvalPrediction 
from glue_data_huggingface import GlueDataset
from glue_data_huggingface import GlueDataTrainingArguments as DataTrainingArguments
from glue_processor_huggingface import glue_compute_metrics

from transformers import (
    HfArgumentParser,
    TrainingArguments,
    glue_output_modes,
    set_seed,
)

from trainer_huggingface import Trainer 

# logging.basicConfig(level=logging.INFO)
@dataclass
class ModelArguments:
    """
    Arguments pertaining to which model/config/tokenizer we are going to fine-tune from.
    """

    model_name_or_path: str = field(
        metadata={"help": "Path to pretrained model or model identifier from huggingface.co/models"}
    )
    config_name: Optional[str] = field(
        default=None, metadata={"help": "Pretrained config name or path if not the same as model_name"}
    )
    tokenizer_name: Optional[str] = field(
        default=None, metadata={"help": "Pretrained tokenizer name or path if not the same as model_name"}
    )
    cache_dir: Optional[str] = field(
        default=None, metadata={"help": "Where do you want to store the pretrained models downloaded from s3"}
    )
def compute_metrics(p: EvalPrediction) -> Dict:
    '''
    Task specific way of computing metrics
    '''
    preds = np.argmax(p.predictions, axis=1)
    
    return glue_compute_metrics('dnc2', preds, p.label_ids)

def str2bool(v):
    return v.lower() in ("yes", "true", "t", "1")<jupyter_output><empty_output><jupyter_text>Load roberta-large-mnli and load trainer class<jupyter_code>data_dir = "/data/sid/temporal_nli/data/"
model_dir = "/data/sid/temporal_nli/saved_models/roberta-large/"
def save_model_predictions(dev_file, 
                                test_file,
                                model_path,
                                dev_dir = "../../../data/dev/",
                                test_dir = "../../../data/test/",
                                hypo_only=False,
                            results_location="results/"):
    '''
    Given model path, make predictions json file for dev and test
    
    Parameters
    ----------
    dev_file: Name of the dev dataset tsv file
    test_file: Name of the test dataset tsv file
    model_path: Directory location of the model path. Directory contains
                pytorch_model.bin, config.json etc.
    dev_dir: directory location of dev dataset
    test_dir: directory location of test dataset
    hypo_only: Boolean for whether it is a hpothesis_only model
    results_location: Directory location where results are to be stored
    '''
    os.makedirs(results_location, exist_ok=True)
    
    model_args = ModelArguments(model_name_or_path=model_path)
    num_labels=2
    print(f"Model path: {model_path}")

    ## Load configs, tokenizer, model
    config = AutoConfig.from_pretrained(
                    model_args.model_name_or_path,
                    num_labels=num_labels,
                    finetuning_task="dnc2",)
    tokenizer = AutoTokenizer.from_pretrained(
                model_args.model_name_or_path,)
    model = AutoModelForSequenceClassification.from_pretrained(
                    model_args.model_name_or_path,
                    config=config,)
    
    ## Dummy training Arguments -- not relevant for predictions
    training_args = TrainingArguments(output_dir = ".models/model_name/",
                                 save_total_limit=8,
                                  per_gpu_eval_batch_size=128,
                                 )
    
    ## Initialize dummy trainer
    trainer = Trainer(
            model=model,
            args=training_args,
            train_dataset=None,
            eval_dataset=None,
            compute_metrics=compute_metrics,
    )
    

    ### Load datasets
    # Extract features from dataset
    dev_args = DataTrainingArguments(task_name="dnc2", data_dir=dev_dir)
    test_args = DataTrainingArguments(task_name="dnc2", data_dir=test_dir)
    
    dev_dataset = GlueDataset(dev_args, data_filename=dev_file,
                               num_labels = num_labels,
                               tokenizer=tokenizer,
                              hypothesis_only=hypo_only)

    test_dataset = GlueDataset(test_args, data_filename=test_file,
                               num_labels = num_labels,
                               tokenizer=tokenizer,
                              hypothesis_only=hypo_only)

    ### Predictions
    dev_preds = trainer.predict(dev_dataset)
    test_preds = trainer.predict(test_dataset)
    
    pred_dict = {0: 'not-entailed', 1:'entailed'}
    
    results_filename = model_args.model_name_or_path.split("/")[-3] + "--" + model_args.model_name_or_path.split("/")[-2]
    
    ## Dev
    df_dev = pd.read_csv(dev_args.data_dir + dev_file, sep='\t')
    df_dev['predicted_label'] = [pred_dict[x] for x in np.argmax(dev_preds[0], axis=1)]
    df_dev = df_dev.rename(columns={'label': 'true_label'})
    
    ## Test
    df_test = pd.read_csv(test_args.data_dir + test_file, sep='\t')
    df_test['predicted_label'] = [pred_dict[x] for x in np.argmax(test_preds[0], axis=1)]
    df_test = df_test.rename(columns={'label': 'true_label'})
    
    ### JSON Files
    ## Dev
    with open("results/" + results_filename + "--dev.json", "w") as f1:
        json.dump(df_dev.to_dict(orient='index'), f1, indent=4)
        
    ## Test
    with open("results/" + results_filename + "--test.json", "w") as f1:
        json.dump(df_test.to_dict(orient='index'), f1, indent=4)
        
    ### TXT files metrics
    ## Dev
    with open("results/" + results_filename + "--dev.txt", "w") as f1:
        json.dump(dev_preds.metrics, f1, indent=4)
        
    ## Test
    with open("results/" + results_filename + "--test.txt", "w") as f1:
        json.dump(test_preds.metrics, f1, indent=4)
    
    print(f"dev metrics: {dev_preds.metrics}")
    print(f"test metrics: {test_preds.metrics}")
    
    return None<jupyter_output><empty_output><jupyter_text>Duration Models<jupyter_code>hypo_true = "../../../saved_models/roberta-large/roberta-large_uds_dur_lr_2e-05_wt_0.1_ws_122_hypo_only_True/checkpoint-13500/"
hypo_false = "../../../saved_models/roberta-large/roberta-large_uds_dur_lr_2e-05_wt_0.1_ws_122_hypo_only_False/checkpoint-6000/"
## Hypo_True
save_model_predictions("dev-temporal-duration-data.tsv", 
                        "test-temporal-duration-data.tsv",
                        hypo_true,
                       hypo_only=True,
                        dev_dir = "../../../data/dev/",
                        test_dir = "../../../data/test/",
                        results_location="results/")
## Hypo_False
save_model_predictions("dev-temporal-duration-data.tsv", 
                        "test-temporal-duration-data.tsv",
                        hypo_false,
                        hypo_only=False,
                        dev_dir = "../../../data/dev/",
                        test_dir = "../../../data/test/",
                        results_location="results/")<jupyter_output>Model path: ../../../saved_models/roberta-large/roberta-large_uds_dur_lr_2e-05_wt_0.1_ws_122_hypo_only_False/checkpoint-6000/<jupyter_text>UDS-Rel<jupyter_code>hypo_true = "../../../saved_models/roberta-large/roberta-large_uds_rel_lr_2e-05_wt_0.1_ws_122_hypo_only_True/checkpoint-5000/"
hypo_false = "../../../saved_models/roberta-large/roberta-large_uds_rel_lr_2e-05_wt_0.1_ws_122_hypo_only_False/checkpoint-10000/"
## Hypo_True
save_model_predictions("dev-temporal-relation-data.tsv", 
                        "test-temporal-relation-data.tsv",
                        hypo_true,
                       hypo_only=True,
                        dev_dir = "../../../data/dev/",
                        test_dir = "../../../data/test/",
                        results_location="results/")

## Hypo_True
save_model_predictions("dev-temporal-relation-data.tsv", 
                        "test-temporal-relation-data.tsv",
                        hypo_false,
                       hypo_only=False,
                        dev_dir = "../../../data/dev/",
                        test_dir = "../../../data/test/",
                        results_location="results/")<jupyter_output>Model path: ../../../saved_models/roberta-large/roberta-large_uds_rel_lr_2e-05_wt_0.1_ws_122_hypo_only_True/checkpoint-5000/<jupyter_text>TB-Dense<jupyter_code>hypo_true = "../../../saved_models/roberta-large/roberta-large_tbdense_lr_2e-05_wt_0.1_ws_122_hypo_only_True/checkpoint-1392/"
hypo_false = "../../../saved_models/roberta-large/roberta-large_tbdense_lr_2e-05_wt_0.1_ws_122_hypo_only_False/checkpoint-2320/"
## Hypo_True
save_model_predictions("dev-tbdense-data.tsv", 
                        "test-tbdense-data.tsv",
                        hypo_true,
                       hypo_only=True,
                        dev_dir = "../../../data/dev/",
                        test_dir = "../../../data/test/",
                        results_location="results/")

## Hypo_True
save_model_predictions("dev-tbdense-data.tsv", 
                        "test-tbdense-data.tsv",
                        hypo_false,
                       hypo_only=False,
                        dev_dir = "../../../data/dev/",
                        test_dir = "../../../data/test/",
                        results_location="results/")<jupyter_output>Model path: ../../../saved_models/roberta-large/roberta-large_tbdense_lr_2e-05_wt_0.1_ws_122_hypo_only_True/checkpoint-1392/<jupyter_text>TE3<jupyter_code>hypo_true = "../../../saved_models/roberta-large/roberta-large_te3_lr_2e-05_wt_0.1_ws_122_hypo_only_True/checkpoint-6435/"
hypo_false = "../../../saved_models/roberta-large/roberta-large_te3_lr_2e-05_wt_0.1_ws_122_hypo_only_False/checkpoint-1287/"
## Hypo_True
save_model_predictions("dev-tempeval3-data.tsv", 
                        "test-tempeval3-data.tsv",
                        hypo_true,
                       hypo_only=True,
                        dev_dir = "../../../data/dev/",
                        test_dir = "../../../data/test/",
                        results_location="results/")

## Hypo_True
save_model_predictions("dev-tempeval3-data.tsv", 
                        "test-tempeval3-data.tsv",
                        hypo_false,
                       hypo_only=False,
                        dev_dir = "../../../data/dev/",
                        test_dir = "../../../data/test/",
                        results_location="results/")<jupyter_output>Model path: ../../../saved_models/roberta-large/roberta-large_te3_lr_2e-05_wt_0.1_ws_122_hypo_only_True/checkpoint-6435/<jupyter_text>RED<jupyter_code>hypo_true = "../../../saved_models/roberta-large/roberta-large_red_lr_2e-05_wt_0.1_ws_122_hypo_only_True/checkpoint-303/"
hypo_false = "../../../saved_models/roberta-large/roberta-large_red_lr_2e-05_wt_0.1_ws_122_hypo_only_False/checkpoint-1515/"
## Hypo_True
save_model_predictions("dev-red-data.tsv", 
                        "test-red-data.tsv",
                        hypo_true,
                       hypo_only=True,
                        dev_dir = "../../../data/dev/",
                        test_dir = "../../../data/test/",
                        results_location="results/")

## Hypo_True
save_model_predictions("dev-red-data.tsv", 
                        "test-red-data.tsv",
                        hypo_false,
                       hypo_only=False,
                        dev_dir = "../../../data/dev/",
                        test_dir = "../../../data/test/",
                        results_location="results/")<jupyter_output>Model path: ../../../saved_models/roberta-large/roberta-large_red_lr_2e-05_wt_0.1_ws_122_hypo_only_True/checkpoint-303/
{'fraction_non_alphanumeric': 0.12512470901230463, 'fraction_numerical': 0.018373794479547723, 'mean_word_length': 2.0391611925214757, 'pattern_counts': {'":': 4, '<': 24, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '209646', 'license': 'MIT', 'n_tokens_mistral': 4014, 'n_tokens_neox': 3793, 'n_words': 550, 'path': 'src/models/roberta/Evaluate_models.ipynb', 'repo_name': 'sidsvash26/temporal_nli', 'size': '12028'}