<jupyter_start><jupyter_text>Qualitative Analysis 1<jupyter_code>import pandas as pd
import os
import re
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from tqdm import tqdm
from IPython.display import display, HTML, Markdown
from lr.text_processing.util import pre_process_nli_df
from collections import Counter
from lr.stats.h_testing import get_paired_t_statistic<jupyter_output><empty_output><jupyter_text>When $\rho=0$ which test examples are affected by the transformation?<jupyter_code>m_name = "roberta_large"
data = "snli"
batch = 1
path = "raw_results/{}/{}/syn_p_h/batch{}/rho_0p00_results.csv".format(data, m_name, batch)
df = pd.read_csv(path)

snli_trans = pd.read_csv("data/{}/test_p_h_syn_noun.csv".format(data))
sin_noun = pd.read_csv("data/{}/syn_noun.csv".format(data))
key2id = {k:i for k,i in zip(sin_noun.key,sin_noun.index)}
f = lambda k: key2id[k]
def filter_df_by_label(df, drop_label='-'):
    """
    drop observations with label 'drop_label'
    """
    return df.loc[df.label != drop_label]
snli_ori = pd.read_csv("data/{}/test.csv".format(data))
snli_ori = filter_df_by_label(snli_ori)
pre_process_nli_df(snli_ori)
snli_ori = snli_ori.reset_index(drop=True)
t = get_paired_t_statistic(df)
t
def e(alpha, n):
    return np.sqrt(np.log(2/alpha) / (2*n))

ids = df.loc[(df.A==1) & (df.B==0)]
ids = ids.index.tolist()

b = [44, 32, 80, 308,
     331, 463, 574,
     594, 677, 712, 877,
     910, 946, 982, 983, 1233, 1246,
     1270, 1321, 1353, 1389, 1402,
     1500, 1703, 1766, 1920, 2015,
     2209, 2240, 2375, 2380, 2446,
     2477, 2683, 2695, 2738, 2911, 3034,
     3092, 3279, 3332, 3388, 3470, 3555,
     3559, 3650, 3764]

b_to_print = [677, 982, 1312, 1353, 2015, 2240, 2446] 
g = [i for i in ids[:ids.index(3764)] if i not in b]
g_to_print = [1613, 1827, 2058, 1784, 2412, 1613]

end_ = ids[:ids.index(3764)]

more = ids[ids.index(3764):]

gn = len(g)
bn = len(b)
pct = gn/(gn+bn)

n = gn+bn 
n = 200


e95 = lambda n: e(0.05, n)
x = np.array(range(1,n+1))

plt.plot(x, e95(x),color="blue");
p = np.random.uniform(0.5,1)
eps = e95(x)
n = eps.argmin()
e_id = eps[n]


print("n = {} | e_n = {:.1%}".format(x[n], e_id))


print(pct,"[",pct-e_id,",", pct+e_id, "]")
ids = df.loc[(df.A==1) & (df.B==0)]
ids = ids.index.tolist()
n = len(ids)
pct = n/df.shape[0]
display(Markdown(r"### When $\rho=0.0$, the number of disrupted sentences (1 to 0) is {} ({:.1%} of the test data)".format(n,pct)))
comb = pd.concat([snli_ori, snli_trans.add_suffix("_2")],1)
comb = comb.loc[more] 
display(HTML(comb.head(3).to_html()))
print(g)
print(b)<jupyter_output>[44, 32, 80, 308, 331, 463, 574, 594, 677, 712, 877, 910, 946, 982, 983, 1233, 1246, 1270, 1321, 1353, 1389, 1402, 1500, 1703, 1766, 1920, 2015, 2209, 2240, 2375, 2380, 2446, 2477, 2683, 2695, 2738, 2911, 3034, 3092, 3279, 3332, 3388, 3470, 3555, 3559, 3650, 3764]<jupyter_text>What are the most commom transformations that appear on the error pairs?<jupyter_code>spaces = re.compile(' +')
results = []

ids = comb.index.tolist()
for id_ in ids:
    s = comb.loc[id_].premise + " " + comb.loc[id_].hypothesis
    st = comb.loc[id_].premise_2 + " " + comb.loc[id_].hypothesis_2
    s = spaces.sub(" ",s)
    st = spaces.sub(" ",st)
    s = s.strip()
    st = st.strip()
    s = set(s.split(" "))
    st = set(st.split(" "))
    diff = list(s - st)
    try:
        diff = list(map(f, diff))
        results += diff
    except KeyError:
        print(id_)
        print(s)
        print(st)
    
    
    
results_c = Counter(results)
dict_id = [i[0] for i in results_c.most_common(10)]
sin_noun.loc[:, "freq"] = [results_c[i]/len(results) for i in sin_noun.index]
sin_noun = sin_noun.sort_values("freq", ascending=False)
sin_noun.loc[:, "freq"] = sin_noun.freq.map(lambda x: "{:.1%}".format(x))
sin_noun.head(10)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17567567567567569, 'fraction_numerical': 0.1172037422037422, 'mean_word_length': 4.152610441767068, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '94452', 'license': 'MIT', 'n_tokens_mistral': 2004, 'n_tokens_neox': 1666, 'n_words': 378, 'path': 'src/qualitative1.ipynb', 'repo_name': 'lukewanless/looking-for-equivalences', 'size': '3848'}