<jupyter_start><jupyter_text>Import<jupyter_code>%matplotlib widget
import pickle
import os
import pandas as pd
import numpy as np
import json

import matplotlib.pyplot as plt
import seaborn as sns

from sklearn.metrics import roc_auc_score, roc_curve<jupyter_output><empty_output><jupyter_text>Define<jupyter_code>def compute_px(ppl, lls):
    lengths = np.array([len(ll) for ll in lls])
    logpx = np.log(ppl) * lengths * -1
    return logpx

def compute_auroc_all(id_msp, id_px, id_ppl, ood_msp, ood_px, ood_ppl, do_print=False):
    score_px = compute_auroc(-id_px, -ood_px)
    score_py = compute_auroc(-id_msp, -ood_msp)
    score_ppl = compute_auroc(id_ppl, ood_ppl)
    if do_print:
        print(f"P(x): {score_px:.3f}")
        print(f"P(y | x): {score_py:.3f}")
        print(f"Perplexity: {score_ppl:.3f}")
    scores = {
        'p_x': score_px,
        'p_y': score_py,
        'ppl': score_ppl
    }
    return scores
def compute_auroc(id_pps, ood_pps, normalize=False, return_curve=False):
    y = np.concatenate((np.ones_like(ood_pps), np.zeros_like(id_pps)))
    scores = np.concatenate((ood_pps, id_pps))
    if normalize:
        scores = (scores - scores.min()) / (scores.max() - scores.min())
    if return_curve:
        return roc_curve(y, scores)
    else:
        return 100*roc_auc_score(y, scores)

def compute_far(id_pps, ood_pps, rate=5, return_indices=False):
    if return_indices:
        cut_off = np.percentile(ood_pps, rate)
        id_indices = [i for i, pps in enumerate(id_pps) if pps > cut_off]
        ood_indices = [i for i, pps in enumerate(ood_pps) if pps < cut_off]
        return {'id': id_indices, 'ood': ood_indices}
    else:
        incorrect = len(id_pps[id_pps > np.percentile(ood_pps, rate)])
        return 100*incorrect / len(id_pps)
def compute_metric_all(id_msp, id_px, id_ppl, ood_msp, ood_px, ood_ppl, metric='auroc', do_print=False):
    if metric == 'auroc':
        score_px = compute_auroc(-id_px, -ood_px)
        score_py = compute_auroc(-id_msp, -ood_msp)
        score_ppl = compute_auroc(id_ppl, ood_ppl)
    elif metric == 'far':
        score_px = compute_far(-id_px, -ood_px)
        score_py = compute_far(-id_msp, -ood_msp)
        score_ppl = compute_far(id_ppl, ood_ppl)
    else:
        raise Exception('Invalid metric name')

    if do_print:
        print(f"Metric {metric}:")
        print(f"P(x): {score_px:.3f}")
        print(f"P(y | x): {score_py:.3f}")
        print(f"Perplexity: {score_ppl:.3f}\n")

    scores = {
        'p_x': score_px,
        'p_y': score_py,
        'ppl': score_ppl
    }
    return scores
def read_model_out(fname):
    ftype = fname.split('.')[1]
    
    if ftype == 'pkl':
        with open(fname, 'rb') as f:
            return pickle.load(f)
    elif ftype == 'npy':
        return np.load(fname)
    else:
        raise KeyError(f'{ftype} not supported')<jupyter_output><empty_output><jupyter_text>Summarize  Presettings<jupyter_code>verbose = False
repo = os.path.dirname(os.path.dirname(os.path.abspath('__file__')))
print(repo)
output_dir = os.path.join(repo, 'output')
fig_dir = os.path.join(repo, 'figs')
train_sets = ['imdb', 'sst2']
eval_sets = ['imdb', 'sst2', 'snli', 'counterfactual-imdb', 'rte']
methods = ['msp', 'lls', 'pps']

signals = {}
for train_set in train_sets:
    for eval_set in eval_sets:
        signals[(train_set, eval_set)] = {method: None for method in methods}<jupyter_output><empty_output><jupyter_text>Import Signals<jupyter_code>method2ftype={
    'msp': 'npy',
    'lls': 'pkl',
    'pps': 'npy',
}<jupyter_output><empty_output><jupyter_text>Subsampling Indices<jupyter_code>def get_indices(fname):
    with open(fname, 'r') as f:
        return [int(x) for x in f.readlines()]
roberta_dir = os.path.join(repo, 'roberta')

subsample_indices = {
    data_name: get_indices(os.path.join(roberta_dir, f'{data_name}_indices.txt'))
    for data_name in train_sets
}<jupyter_output><empty_output><jupyter_text>GPT2<jupyter_code>best_lr = {
    'imdb': '5e-5',
    'sst2': '5e-5',
}

methods = ['lls', 'pps']
not_readys = []

for (train_set, eval_set), signals_dict in signals.items():
    for method in methods:
        signal_fname = os.path.join(output_dir, 'gpt2', train_set, f'{eval_set}_{best_lr[train_set]}_{method}.{method2ftype[method]}')
        if not os.path.exists(signal_fname):
            not_readys.append((train_set, eval_set, method))
            continue
        
        signal = read_model_out(signal_fname)
        if eval_set in train_sets:
            idxs = subsample_indices[eval_set]
            signal = [signal[idx] for idx in idxs]
        
        signals_dict[method] = signal
        
for not_ready in not_readys:
    print(not_ready)<jupyter_output><empty_output><jupyter_text>RoBERTa<jupyter_code>methods = ['msp']
not_readys = []

model_type = 'roberta-base'

for (train_set, eval_set), signals_dict in signals.items():
    for method in methods:
        signal_fname = os.path.join(output_dir, 'roberta', train_set, f'{model_type}_{eval_set}_{method}.{method2ftype[method]}')
        if not os.path.exists(signal_fname):
            not_readys.append((train_set, eval_set, method))
            continue
            
        signal = read_model_out(signal_fname)
        if train_set != eval_set and eval_set in train_sets:
            idxs = subsample_indices[eval_set]
            signal = np.array([signal[idx] for idx in idxs])
        
        signals_dict[method] = signal
        
for not_ready in not_readys:
    print(not_ready)<jupyter_output><empty_output><jupyter_text>Get Error Indices by FAR95<jupyter_code>score2plot = {
    'p_x': r'GPT2: $p(x)$',
    'ppl': 'GPT2: PPL',
    'p_y': 'RoBERTa: MSP',
}

metric2plot = {
    'auroc': 'AUROC',
    'far': 'FAR95'
}

dataset2plot = {
    'imdb': 'IMDB',
    'sst2': 'SST-2',
    'snli': 'SNLI',
    'counterfactual-imdb': 'c-IMDB',
    'rte': 'RTE',
}

error_indices = {}
not_ready = []
for train_set in train_sets:
    for eval_set in eval_sets:
        if train_set == eval_set:
            continue
        
        ood_signal_dict = signals[(train_set, eval_set)]
        id_signal_dict = signals[(train_set, train_set)]
        
        skip=False
        for value in ood_signal_dict.values():
            if isinstance(value, type(None)):
                skip=True
                
        if skip:
            not_ready.append((train_set, eval_set))
            continue
        
        pps_errors = compute_far(id_signal_dict['pps'], ood_signal_dict['pps'], return_indices=True)
        msp_errors = compute_far(-id_signal_dict['msp'], -ood_signal_dict['msp'], return_indices=True)
        
        error_indices[(train_set, eval_set)] = {'pps': pps_errors, 'msp': msp_errors}
print(list(error_indices.keys()))<jupyter_output>[('imdb', 'sst2'), ('imdb', 'snli'), ('imdb', 'counterfactual-imdb'), ('imdb', 'rte'), ('sst2', 'imdb'), ('sst2', 'snli'), ('sst2', 'counterfactual-imdb'), ('sst2', 'rte')]<jupyter_text>Partition Indices<jupyter_code>indices_parts = {}
for ood_key, errors_dict in error_indices.items():
    pps_id = set(errors_dict['pps']['id'])
    pps_ood = set(errors_dict['pps']['ood'])
    msp_id = set(errors_dict['msp']['id'])
    msp_ood = set(errors_dict['msp']['ood'])
    
    union = {
        'id': pps_id.union(msp_id),
        'ood': pps_ood.union(msp_ood),
    }
    
    common = {
        'id': pps_id.intersection(msp_id),
        'ood': pps_ood.intersection(msp_ood),
    }
    
    pps_only = {
        'id': pps_id.difference(msp_id),
        'ood': pps_ood.difference(msp_ood),
    }
    
    msp_only = {
        'id': msp_id.difference(pps_id),
        'ood': msp_ood.difference(pps_ood),
    }
    
    indices_parts[ood_key] = {
        'union': union,
        'common': common,
        'pps': pps_only,
        'msp': msp_only,
    }<jupyter_output><empty_output><jupyter_text>Partition Stats<jupyter_code>stats = []
for (indomain, ood), partitions in indices_parts.items():
    total_count = len(partitions['union']['id']) + len(partitions['union']['ood'])
    common_count = len(partitions['common']['id']) + len(partitions['common']['ood'])
    msp_count = len(partitions['msp']['id']) + len(partitions['msp']['ood']) + common_count
    pps_count = len(partitions['pps']['id']) + len(partitions['pps']['ood']) + common_count
    
    row = {
        'in domain': indomain,
        'ood': ood,
        'total count': total_count,
        'common count': common_count,
        'msp only count': msp_count - common_count,
        'pps only count': pps_count - common_count,
        'common ratio': common_count/total_count,
        'msp only ratio': (msp_count - common_count)/total_count,
        'pps only ratio': (pps_count - common_count)/total_count,
    }
    
    stats.append(row)
print(pd.DataFrame(stats))
pd.DataFrame(stats).to_csv(os.path.join('.', 'error_analysis', 'error_counts_ratios.csv'))<jupyter_output>in domain                  ood  total count  common count  msp only count  \
0      imdb                 sst2        18063          6200           10932   
1      imdb                 snli        19371          2874             637   
2      imdb  counterfactual-imdb        19920         16083            1408   
3      imdb                  rte        19770          4077              48   
4      sst2                 imdb         2429           177            1362   
5      sst2                 snli         1261            97             600   
6      sst2  counterfactual-imdb          723            50             551   
7      sst2                  rte          363            85             173   

   pps only count  common ratio  msp only ratio  pps only ratio  
0             931      0.343243        0.605215        0.051542  
1           15860      0.148366        0.032884        0.818750  
2            2429      0.807380        0.070683        0.121938  
3           15645      0[...]<jupyter_text>Sample Examples<jupyter_code>with open(os.path.join('.', 'all_val_data.p'), 'rb') as f:
    datasets = pickle.load(f)
seed = 42
np.random.seed(seed)
nsample = 10
examples = []
ignore_ood = []
parts = ['common', 'msp', 'pps']
domains = ['id', 'ood']

for (indomain, ood), partitions in indices_parts.items():
    if ood in ignore_ood:
        continue
        
    data = {
        'id': datasets[('id', 'val', indomain)]['text'],
        'ood': datasets[('ood', 'val', ood)]['text'],
    }
    
    for part in parts:
        for domain in domains:
            sample_size = min(nsample, len(partitions[part][domain]))
            sample = np.random.choice(list(partitions[part][domain]), size=sample_size, replace=False)
            print(indomain, ood, part, domain)
            for idx in sample:
                examples.append({
                    'in domain': indomain,
                    'ood': ood,
                    'text': data[domain][idx],
                    'domain': domain,
                    'dataset': indomain if domain == 'id' else ood,
                    'partition': part,
                })
pd.DataFrame(examples).to_csv(os.path.join('.', 'error_analysis', 'error_examples.csv'))<jupyter_output>imdb sst2 common id
imdb sst2 common ood
imdb sst2 msp id
imdb sst2 msp ood
imdb sst2 pps id
imdb sst2 pps ood
imdb snli common id
imdb snli common ood
imdb snli msp id
imdb snli msp ood
imdb snli pps id
imdb snli pps ood
imdb counterfactual-imdb common id
imdb counterfactual-imdb common ood
imdb counterfactual-imdb msp id
imdb counterfactual-imdb msp ood
imdb counterfactual-imdb pps id
imdb counterfactual-imdb pps ood
imdb rte common id
imdb rte common ood
imdb rte msp id
imdb rte msp ood
imdb rte pps id
imdb rte pps ood
sst2 imdb common id
sst2 imdb common ood
sst2 imdb msp id
sst2 imdb msp ood
sst2 imdb pps id
sst2 imdb pps ood
sst2 snli common id
sst2 snli common ood
sst2 snli msp id
sst2 snli msp ood
sst2 snli pps id
sst2 snli pps ood
sst2 counterfactual-imdb common id
sst2 counterfactual-imdb common ood
sst2 counterfactual-imdb msp id
sst2 counterfactual-imdb msp ood
sst2 counterfactual-imdb pps id
sst2 counterfactual-imdb pps ood
sst2 rte common id
sst2 rte common ood
sst2 rte m[...]<jupyter_text>Consistent OOD Examples<jupyter_code>seed = 42
np.random.seed(seed)
import string
def normalize_text(s):
    """Lower text and remove punctuation, articles and extra whitespace."""
    def remove_articles(text):
        regex = re.compile(r'\b(a|an|the)\b', re.UNICODE)
        return re.sub(regex, ' ', text)
    def white_space_fix(text):
        return ' '.join(text.split())
    def remove_punc(text):
        exclude = set(string.punctuation)
        return ''.join(ch for ch in text if ch not in exclude)
    def lower(text):
        return text.lower()
    #return white_space_fix(remove_articles(remove_punc(lower(s))))
    return white_space_fix(remove_punc(lower(s)))


def get_tokens(s):
    if not s: return []
    return normalize_text(s).split()
ood_stats, ood_examples = [], []

ignore_tasks = []
parts = ['common', 'msp', 'pps']
domains = ['ood']

nsample = 10

for (indomain, ood), partitions in indices_parts.items():
    if ood in ignore_tasks:
        continue
    
    total_count = len(partitions['union']['ood'])
    common_count = len(partitions['common']['ood'])
    msp_count = len(partitions['msp']['ood']) + common_count
    pps_count = len(partitions['pps']['ood']) + common_count
    
    row = {
        'in domain': indomain,
        'ood': ood,
        'total count': total_count,
        'common count': common_count,
        'msp only count': msp_count - common_count,
        'pps only count': pps_count - common_count,
        'common ratio': common_count/total_count,
        'msp only ratio': (msp_count - common_count)/total_count,
        'pps only ratio': (pps_count - common_count)/total_count,
    }
    
    ood_stats.append(row)
    
    
    data = {'ood': datasets[('ood', 'val', ood)]}
    for part in parts:
        for domain in domains:
            sample_size = min(nsample, len(partitions[part][domain]))
            sample = np.random.choice(list(partitions[part][domain]), size=sample_size, replace=False)
            print(indomain, ood, part, domain)
            for idx in sample:
                ood_examples.append({
                    'in domain': indomain,
                    'ood': ood,
                    'text': data[domain]['text'][idx],
                    'unigram length': len(get_tokens(data[domain]['text'][idx])),
                    'label': data[domain]['label'][idx],                    
                    'domain': domain,
                    'dataset': indomain if domain == 'id' else ood,
                    'partition': part,
                })
    
    
pd.DataFrame(ood_stats).to_csv(os.path.join('.', 'error_analysis', 'ood_error_counts_ratios.csv'))
pd.DataFrame(ood_examples).to_csv(os.path.join('.', 'error_analysis', 'ood_error_examples.csv'))<jupyter_output>imdb sst2 common ood
imdb sst2 msp ood
imdb sst2 pps ood
imdb snli common ood
imdb snli msp ood
imdb snli pps ood
imdb counterfactual-imdb common ood
imdb counterfactual-imdb msp ood
imdb counterfactual-imdb pps ood
imdb rte common ood
imdb rte msp ood
imdb rte pps ood
sst2 imdb common ood
sst2 imdb msp ood
sst2 imdb pps ood
sst2 snli common ood
sst2 snli msp ood
sst2 snli pps ood
sst2 counterfactual-imdb common ood
sst2 counterfactual-imdb msp ood
sst2 counterfactual-imdb pps ood
sst2 rte common ood
sst2 rte msp ood
sst2 rte pps ood<jupyter_text>Error<jupyter_code>all_ood_examples = []

ignore_tasks = []
parts = ['common', 'msp', 'pps']
domains = ['ood']
nli = ['snli', 'rte']

nsample = 10

for (indomain, ood), partitions in indices_parts.items():  
    
    data = {'ood': datasets[('ood', 'val', ood)]}
    for part in parts:
        for domain in domains:
            for idx in list(partitions[part][domain]):
                overlap = 0
                if ood in nli:
                    for punct in [ '.', '!', '?']:
                        text_list = data[domain]['text'][idx].split(punct)
                        if len(text_list) > 1:
                            break
                    if text_list[-1] == '':
                        prem, hyp = ' '.join(text_list[:-2]), text_list[-2]
                    else:
                        prem, hyp = ' '.join(text_list[:-1]), text_list[-1]
                    prem = set(prem.split(' '))
                    hyp = set(hyp.split(' '))
                    overlap = len(hyp.intersection(prem)) / len(hyp)
                
                all_ood_examples.append({
                    'in domain': indomain,
                    'ood': ood,
                    'text': data[domain]['text'][idx],
                    'unigram length': len(get_tokens(data[domain]['text'][idx])),
                    'label': data[domain]['label'][idx],                    
                    'domain': domain,
                    'dataset': indomain if domain == 'id' else ood,
                    'partition': part,
                    'overlap': overlap
                })            
all_ood_examples_df = pd.DataFrame(all_ood_examples)<jupyter_output><empty_output><jupyter_text>Lengths<jupyter_code>keeps = ['in domain', 'ood', 'unigram length', 'partition']
groupby = ['in domain', 'ood', 'partition']

grouped_lengths = all_ood_examples_df[keeps].groupby(groupby)
lengths_summary = grouped_lengths.mean()
lengths_summary['std'] = grouped_lengths.std()['unigram length']
lengths_summary = lengths_summary.rename(columns = {'unigram length':'mean'})
print(lengths_summary)
lengths_summary.to_csv(os.path.join('.', 'error_analysis', 'ood_error_lengths_stats.csv'))<jupyter_output><empty_output><jupyter_text>Labels<jupyter_code>keeps = ['in domain', 'ood', 'label', 'partition', 'unigram length']
groupby = ['in domain', 'ood', 'partition', 'label']

grouped_labels = all_ood_examples_df[keeps].groupby(groupby)
keeps = ['in domain', 'ood', 'partition', 'unigram length']
groupby = ['in domain', 'ood', 'partition']

grouped_counts = all_ood_examples_df[keeps].groupby(groupby)
grouped_counts_df = grouped_counts.count().rename(columns = {'unigram length': 'total'})
print(grouped_counts_df)
labels_summary = grouped_labels.count().rename(columns = {'unigram length': 'counts'})
print(labels_summary)
labels_total_summary = labels_summary.join(grouped_counts_df, on=['in domain', 'ood', 'partition'], how='left')
labels_total_summary['ratio'] = labels_total_summary['counts'] / labels_total_summary['total']
print(labels_total_summary)
labels_total_summary.to_csv(os.path.join('.', 'error_analysis', 'ood_error_labels_stats.csv'))<jupyter_output><empty_output><jupyter_text>Overlap<jupyter_code>keeps = ['in domain', 'ood', 'overlap', 'partition']
groupby = ['in domain', 'ood', 'partition']

grouped_overlaps = all_ood_examples_df[keeps].groupby(groupby)
overlaps_summary = grouped_overlaps.mean()
overlaps_summary['std'] = grouped_overlaps.std()['overlap']
overlaps_summary = overlaps_summary.rename(columns = {'overlap':'mean'})
print(overlaps_summary)
overlaps_summary.to_csv(os.path.join('.', 'error_analysis', 'ood_error_overlaps_stats.csv'))<jupyter_output><empty_output><jupyter_text>Check Signals<jupyter_code>print('sst2 in domain', 'msp', len(signals[('sst2', 'sst2')]['msp']))
print('sst2 out domain', 'msp',len(signals[('imdb', 'sst2')]['msp']))
print('sst2 in domain', 'pps',len(signals[('sst2', 'sst2')]['pps']))
print('sst2 out domain', 'pps',len(signals[('imdb', 'sst2')]['pps']))
print('='*45)
print('imdb in domain', 'msp', len(signals[('imdb', 'imdb')]['msp']))
print('imdb out domain', 'msp',len(signals[('sst2', 'imdb')]['msp']))
print('imdb in domain', 'pps',len(signals[('imdb', 'imdb')]['pps']))
print('imdb out domain', 'pps',len(signals[('sst2', 'imdb')]['pps']))
print('snli', 'sst2 in domain', 'msp', len(signals[('sst2', 'snli')]['msp']))
print('snli', 'imdb in domain', 'msp',len(signals[('imdb', 'snli')]['msp']))
print('='*45)
print('rte', 'sst2 in domain', 'msp', len(signals[('sst2', 'rte')]['msp']))
print('rte', 'imdb in domain', 'msp',len(signals[('imdb', 'rte')]['msp']))
print('snli', 'sst2 in domain', 'pps', len(signals[('sst2', 'snli')]['pps']))
print('snli', 'imdb in domain', 'pps',len(signals[('imdb', 'snli')]['pps']))
print('='*45)
print('rte', 'sst2 in domain', 'pps', len(signals[('sst2', 'rte')]['pps']))
print('rte', 'imdb in domain', 'pps',len(signals[('imdb', 'rte')]['pps']))<jupyter_output>snli sst2 in domain pps 10000
snli imdb in domain pps 10000
=============================================
rte sst2 in domain pps 277
rte imdb in domain pps 277
{'fraction_non_alphanumeric': 0.1449458396075193, 'fraction_numerical': 0.016078107543595472, 'mean_word_length': 2.986831913245546, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 67, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '463590', 'license': 'MIT', 'n_tokens_mistral': 7595, 'n_tokens_neox': 7306, 'n_words': 1555, 'path': 'dataset_analyses/sample_error_examples.ipynb', 'repo_name': 'ambujojha/ood-detection', 'size': '20587'}