<jupyter_start><jupyter_text>Metrics - SST2 (id)  Setup<jupyter_code>from sklearn.metrics import roc_auc_score, roc_curve
import numpy as np

def compute_auroc(id_pps, ood_pps, normalize=False, return_curve=False):
    y = np.concatenate((np.ones_like(ood_pps), np.zeros_like(id_pps)))
    scores = np.concatenate((ood_pps, id_pps))
    if normalize:
        scores = (scores - scores.min()) / (scores.max() - scores.min())
    if return_curve:
        return roc_curve(y, scores)
    else:
        return 100*roc_auc_score(y, scores)

def compute_far(id_pps, ood_pps, rate=5):
    incorrect = len(id_pps[id_pps > np.percentile(ood_pps, rate)])
    return 100*incorrect / len(id_pps)
import pandas as pd

columns = ['roberta_type', 'id_ood', 'AUROC', 'FAR']
df = pd.DataFrame(columns=columns)
df<jupyter_output><empty_output><jupyter_text>SST2<jupyter_code>sst2_base = np.load('msp/sst2_in_domain/base_sst2_msp.npy')
sst2_large = np.load('msp/sst2_in_domain/large_sst2_msp.npy')<jupyter_output><empty_output><jupyter_text>IMDB (ood)  Base<jupyter_code>imdb_base = np.load('msp/sst2_in_domain/base_imdb_msp.npy')
auroc_base = compute_auroc(sst2_base, imdb_base)
far_base = compute_far(sst2_base, imdb_base)

print('auroc:', auroc_base)
print('far:', far_base)
idx = 0
df.loc[idx]=['base', 'sst2_imdb', auroc_base, far_base]
df<jupyter_output><empty_output><jupyter_text>Large<jupyter_code>imdb_large = np.load('msp/sst2_in_domain/large_imdb_msp.npy')
auroc_large = compute_auroc(sst2_large, imdb_large)
far_large = compute_far(sst2_large, imdb_large)

print('auroc:', auroc_large)
print('far:', far_large)
idx += 1
df.loc[idx]=['large', 'sst2_imdb', auroc_base, far_base]
df<jupyter_output><empty_output><jupyter_text>Counterfactual-IMDB (ood)  Base<jupyter_code>counterfactual_imdb_base = np.load('msp/sst2_in_domain/base_counterfactual-imdb_msp.npy')
auroc_base = compute_auroc(sst2_base, counterfactual_imdb_base)
far_base = compute_far(sst2_base, counterfactual_imdb_base)

print('auroc:', auroc_base)
print('far:', far_base)
idx += 1
df.loc[idx]=['base', 'sst2_counterfactual-imdb', auroc_base, far_base]
df<jupyter_output><empty_output><jupyter_text>Large<jupyter_code>counterfactual_imdb_large = np.load('msp/sst2_in_domain/large_counterfactual-imdb_msp.npy')
auroc_large = compute_auroc(sst2_large, counterfactual_imdb_large)
far_large = compute_far(sst2_large, counterfactual_imdb_large)

print('auroc:', auroc_large)
print('far:', far_large)
idx += 1
df.loc[idx]=['large', 'sst2_counterfactual-imdb', auroc_base, far_base]
df<jupyter_output><empty_output><jupyter_text>SNLI (ood)  Base<jupyter_code>snli_base = np.load('msp/sst2_in_domain/base_snli_msp.npy')
auroc_base = compute_auroc(sst2_base, snli_base)
far_base = compute_far(sst2_base, snli_base)

print('auroc:', auroc_base)
print('far:', far_base)
idx += 1
df.loc[idx]=['base', 'sst2_snli', auroc_base, far_base]
df<jupyter_output><empty_output><jupyter_text>Large<jupyter_code>snli_large = np.load('msp/sst2_in_domain/large_snli_msp.npy')
auroc_large = compute_auroc(sst2_large, snli_large)
far_large = compute_far(sst2_large, snli_large)

print('auroc:', auroc_large)
print('far:', far_large)
idx += 1
df.loc[idx]=['large', 'sst2_snli', auroc_base, far_base]
df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1401840490797546, 'fraction_numerical': 0.013803680981595092, 'mean_word_length': 8.591176470588236, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '60365', 'license': 'MIT', 'n_tokens_mistral': 1460, 'n_tokens_neox': 1447, 'n_words': 179, 'path': 'roberta/notebooks/sst2_metrics.ipynb', 'repo_name': 'ambujojha/ood-detection', 'size': '3260'}