<jupyter_start><jupyter_text>Analyzing Pretrained Representations<jupyter_code>import os
import gc
import json
import torch
import numpy as np
import pandas as pd
import random
import argparse
from tqdm import tqdm
import pandas as pd
from config import config
from utils import seed, create_trainer, update_results
import pytorch_lightning as pl
from evaluation import evaluate
from dataset.loader import create_loaders
from dataset.dataset import create_datasets
from transformers import AutoTokenizer
from Trainer import LightningModel
from models.model import Model
from datasets import load_dataset
datasets = create_datasets(task='imdb_sst2_sa')
models = config['models']
model_name = models[0]
tokenizer = AutoTokenizer.from_pretrained(model_name, usefast=True, use_lower_case=True)
loaders = create_loaders(
        dataset=datasets,
        task='imdb_sst2_sa',
        tokenizer=tokenizer
    )
lm = LightningModel(
    model_name=model_name,
    task="imdb_sst2_sa",
    config=config
)
lm.load_state_dict(
    torch.load("./outputs/imdb_sst2_sa/distilbert-base-uncased/epoch=2.ckpt", map_location=torch.device('cpu'))['state_dict']
)
trainer = pl.Trainer()
trainer.test(
    model=lm,
    test_dataloaders=loaders['sst2']['test']
)
df = pd.read_csv("./outputs/imdb_sst2_sa/sst2.csv")
m = list(map(lambda x:x.split("-")[0]+"_prob", models))
df['mean_prob'] = df[m].mean(axis=1)
df.to_csv("./outputs/imdb_sst2_sa/sst2_mean.csv", index=False)
df.head()
predictions =  load_dataset('csv', data_files='./outputs/imdb_sst2_sa/sst2_mean.csv')['train']
predictions
predictions = predictions.filter(lambda example : example['mean_prob'] < 0.8)
predictions
tokenized = predictions.map(lambda x: tokenizer(x['text'], padding='max_length', truncation=True, max_length=config['tasks']['mnli']['max_seq_length']), batched=True)

tokenized.set_format(type='torch', columns=['input_ids', 'attention_mask'])
# lm.eval()
for param in lm.model.parameters():
    param.requires_grad = False
outputs = torch.empty((0, 768))
for each in tqdm(tokenized):

    _, out = lm(
        input_ids=each['input_ids'].unsqueeze(0), 
        attention_mask=each['attention_mask'].unsqueeze(0),
    )

    outputs = torch.vstack((outputs, out))


    # print(each['input_ids'].shape, each['attention_mask'].shape)

    # print(outputs.shape)
    # break
print(outputs.shape)
torch.save(outputs, "./outputs/imdb_sst2_sa/sst2_finetuned.pt")
# torch.hstack?<jupyter_output><empty_output><jupyter_text>HANS Dataset<jupyter_code>hans = load_dataset("hans")
hans
set(hans['train']['heuristic'])
len(set(hans['train']['template']))
hans['train']['template'][:10]
print(hans['train']['premise'][0], hans['train']['hypothesis'][0])
print(hans['train']['premise'][2], hans['train']['hypothesis'][2])
hans['train']['template'][-10:]
hans['train'][0]
hans['train'][1]
hans['train'][1000]
# hans['train'][999]
df = pd.read_csv("/home/macab/research/robust-representation-learning/concept-explanation/outputs/mnli/mnli_mean.csv")
# df['mean_prob'].quantile(q=0.95)
df_ = df[df['mean_prob']<0.20]
df_['text'].iloc[0]
df_['ground_truth'].iloc[0]
df_['mean_prob'].iloc[0]
df['mean_prob'][0]
set(hans['train']['heuristic'])<jupyter_output><empty_output><jupyter_text>HANS Analysis<jupyter_code>import pandas as pd
import numpy as np
from datasets import load_dataset
hans = load_dataset("hans")
hans
df = pd.read_csv("/home/macab/research/robust-representation-learning/concept-explanation/outputs/hans/hans.csv")
df.head()
df['hypothesis'] = hans['validation']['hypothesis']
df['premise'] = hans['validation']['premise']
df['template'] = hans['validation']['template']
df['heuristic'] = hans['validation']['heuristic']
df.head()
df_ = df[df['distilbert_prob']<0.20]
df_.shape
df_.head()
df_.to_csv("./outputs/hans/bottom_5.csv", index=False)
df_2 = df[df['distilbert_prob']>0.999942]
df_2.shape
df_2.to_csv("./outputs/hans/top_5.csv", index=False)
df_3 = df[df['distilbert_prob']<0.50]
df_3.shape
df_3['heuristic'].value_counts().plot(kind="bar")

df_4 = df[df['distilbert_prob']>0.50]
df_4['heuristic'].value_counts().plot(kind="bar")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17086766844401358, 'fraction_numerical': 0.018904507998061076, 'mean_word_length': 7.2375249500998, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '630179', 'license': 'MIT', 'n_tokens_mistral': 1557, 'n_tokens_neox': 1516, 'n_words': 231, 'path': 'concept-explanation/Results-Analysis.ipynb', 'repo_name': 'macabdul9/robust-representation-learning', 'size': '4126'}