<jupyter_start><jupyter_text>Let :$Y = \{y_1, y_2, y_3, ... , y_n \}$ -> the original label$P^{M_C}_{X_B} = \{p_1, p_2, p_3, ... , p_n \}$ -> the prediction of model $M_C$ on $X_B$$E^{M}_{X_B} = \{id, \quad id \in Y \land id \in P^{M}_{X_B} \land y_{id} \neq p_{id} \}$ -> a list of failed test cases where prediction of the model ${M}$ is incorrect on $X_B$$E^{M_C}_{X_B} = \{id, \quad id \in Y \land id \in P^{M_C}_{X_B} \land y_{id} \neq p_{id} \}$ -> a list of failed test cases where prediction of the robust ${M_C}$ is wrong for $X_B$$Difference(List_1, List_2) = \{ id, \quad id \in List_1 \land id \not \in List_2 \}$$R^C_B = Difference(E^{M}_{X_B}, E^{M_C}_{X_B})$$R^B_B = Difference(E^{M}_{X_B}, E^{M_B}_{X_B})$$Intersection(List_2, List_2) = \{ id, \quad id \in List_1 \land id \in List_2 \} $Inspired from <NAME>, we propose BSEM to measure the similarity of bug-attaccking behavior from adversarial attack$BSEM_{C-fix-B} = \frac{Length(Intersection(R^C_B, R^B_B))}{Length(R^C_B) + Length(R^B_B) - Length(Intersection(R^C_B, R^B_B))}$$BSEM_{B-fix-C} = \frac{Length(Intersection(R^B_C, R^C_C))}{Length(R^B_C) + Length(R^C_C) - Length(Intersection(R^B_C, R^C_C))}$$BSEM(B,C) = \frac{BSEM_{B-fix-C} + BSEM_{C-fix-B}}{2}$Failed Test Cases instead of Robust<jupyter_code>def get_robust_data(train, test):
    y_adv = load_data(train, test, "y_cross_robust")
    y_adv_pred = load_data(train, test, "y_cross_robust_pred")
    return y_adv, y_adv_pred

train = "pgd"
test = "autoattack"
y_adv, y_adv_pred = get_robust_data(train, test)

print("Y_adv({},{}): {}".format(train, test, y_adv))
print("Y_adv_pred({},{}): {}".format(train, test, y_adv_pred))
train = "fgsm"
test = "autopgd"
y_adv, y_adv_pred = get_robust_data(train, test)

print("Y_adv({},{}): {}".format(train, test, y_adv))
print("Y_adv_pred({},{}): {}".format(train, test, y_adv_pred))
def error(l1, l2):
    if len(l1) != len(l2) :
        raise ValueError("The array length must be same")
    check = np.not_equal(l1, l2)
    return np.argwhere(check == True).reshape(-1)

def correct(l1, l2):
    if len(l1) != len(l2) :
        raise ValueError("The array length must be same")
    check = np.equal(l1, l2)
    return np.argwhere(check == True).reshape(-1)    

def repair(l1, l2) :
#     return [x for x in l1 if x not in l2]
    return l1[np.isin(l1, l2, invert=True)]

y1, y1_pred = get_robust_data("original", test)
y2, y2_pred = get_robust_data(train, test)


l1 = [0,1,2] 
l2 = [0,5,2]
error([0,1,2], [0,5,2])
correct(l1, l2)
def get_correct(train, test):
    y, y_pred = get_robust_data(train, test)
    return correct(y, y_pred)
    

def get_distribution(train, test):
    correct_original_test = get_correct("original", test)
    correct_train_test = get_correct(train, test)
    correct_test_test = get_correct(test, test)
    n = len(load_data(train, test, "y_cross_robust"))
    l = np.zeros(n).astype(np.int)
    for i in range(len(l)):
        if i in correct_original_test :
            l[i] = 1
        elif i in correct_train_test :
            l[i] = 2
        elif i in correct_test_test :
            l[i] = 3
        if i in correct_train_test and i in correct_test_test :
            l[i] = 4
            
    unique, counts = np.unique(l, return_counts=True)
    counter = dict(zip(unique, counts))
    index = []
    data = []
    for i in range(5) :
        index.append(i)
        if i in unique :
            data.append(counter[i])
        else :
            data.append(0)
            
#     print(counter)

    return pd.DataFrame(data={"index": index, "count": data})
df = get_distribution("autoattack", "pgd")
sns.barplot(data=df, x="index", y="count")
fig, axes = plt.subplots(2, 2, figsize=(18, 10))

fig.suptitle('Pokemon Stats by Generation')

sns.barplot(ax=axes[0, 0], data=df, x='index', y="count")
sns.barplot(ax=axes[0, 1], data=df, x='index', y="count")
sns.barplot(ax=axes[1, 0], data=df, x='index', y="count")
sns.barplot(ax=axes[1, 1], data=df, x='index', y="count")
distributions = {}
for x1 in attacks :
    d = {}
    for x2 in attacks :
        train = x1
        test = x2
        d[x2] = get_distribution(train, test)
    distributions[x1] = d
m = len(attacks)
# m = 3
fig, axes = plt.subplots(m, m, figsize=(20, 18))

g = {}

for i in range(m) :
    g[i] = {}
    for j in range(m) :
        train = attacks[i]
        test = attacks[j]
        df = pd.DataFrame(data=distributions[train][test])
        axes[i, j].set(ylim=(0, 10000))
        g[i][j] = sns.barplot(ax=axes[i, j], data=df, x='index', y="count")
    
        if i != m-1 :
            g[i][j].set(xticklabels=[])  
            g[i][j].set(xlabel=None)
            g[i][j].tick_params(bottom=False)  # remove the ticks
        else :
            g[i][j].set(xlabel=test)
        
        
        if j != 0 :
            g[i][j].set(yticklabels=[])  
            g[i][j].set(ylabel=None)
            g[i][j].tick_params(top=False)  # remove the ticks
        else :
            g[i][j].set(ylabel=train)

fig.savefig("plot/dist.png", bbox_inches='tight')
R = repair(error(y1, y1_pred), error(y2, y2_pred))
len(R)
from sklearn.utils import resample

x = ["a", "b", "c", "d", "e"]
y = [1, 2, 3, 4, 5]

resample(x, n_samples=2, random_state=0)
resample(y, n_samples=2, random_state=0)
def match(l1, l2) :
#     return [x for x in l1 if x in l2]
    return l1[np.isin(l1, l2)]

len(match(R,R))
def get_repair(train, test): 
    y1, y1_pred = get_robust_data("original", test)
    y2, y2_pred = get_robust_data(train, test)
    R = repair(error(y1, y1_pred), error(y2, y2_pred))
    return R

def is_not_subset(train, test) :
    R_train_test = get_repair(train, test)
    R_test_test = get_repair(test, test)
    
#     return len(repair(R_test_test, R_train_test)) > 0 
    return len(repair(R_train_test, R_test_test)) > 0 

def one_pov_relation(train, test) :
    R_train_test = get_repair(train, test)
    R_test_test = get_repair(test, test)
    
    intersection = len(match(R_train_test, R_test_test))
    union = len(R_train_test) + len(R_test_test) - intersection
    
    return intersection / union

one_pov_relation(train, test)
def BSEM(a1, a2) :
    return (one_pov_relation(a1, a2) + one_pov_relation(a2, a1))/2

BSEM(train, test)
BSEM("pixelattack", "autoattack")
BSEM("squareattack", "autoattack")
BSEM("pgd", "fgsm")
BSEM("cw", "fgsm")
from constant import TOOLBOX_ADV_ATTACK_LIST

# TOOLBOX_ADV_ATTACK_LIST = ["autoattack", "deepfool", "fgm", "pgd", "squareattack"]

TOOLBOX_ADV_ATTACK_LIST = ["autoattack", "autopgd", "bim", "cw", "fgsm", "pgd", "deepfool", "newtonfool", "squareattack", "pixelattack", "spatialtransformation"]
# TOOLBOX_ADV_ATTACK_LIST = ["autoattack", "autopgd", "bim", "cw", "deepfool", "fgsm", "jsma", "newtonfool", "pixelattack", "pgd",  "squareattack"]

attacks = TOOLBOX_ADV_ATTACK_LIST
metrics = {}
for a1 in attacks :
    m = {}
    for a2 in attacks :
        m[a2] = is_not_subset(a1, a2)
    metrics[a1] = m  
    
rq1 = pd.DataFrame(data=metrics)
def plot_half_heatmap(data, cmap, path) :
    sns.set_theme(style="white")

    # Generate a mask for the upper triangle
    mask = np.triu(np.ones_like(different_bugs_metrics, dtype=bool))

    # Set up the matplotlib figure
    f, ax = plt.subplots(figsize=(8, 5))

    # Draw the heatmap with the mask and correct aspect ratio
    f = sns.heatmap(data, mask=mask, cmap=cmap, vmax=1, center=0,
                square=True, linewidths=.5, cbar=False)
    f.figure.savefig(path, bbox_inches='tight')

def plot_heatmap(data, cmap, path, annot=False) :
    sns.set_theme(style="white")

    # Draw the heatmap with the mask and correct aspect ratio
    if annot :
        f, ax = plt.subplots(figsize=(12, 6))
        f = sns.heatmap(data, cmap=cmap, vmax=1, center=0, annot=annot, fmt=".3f",
                        linewidths=.5, cbar_kws={"shrink": .5})
        f.figure.savefig(path, bbox_inches='tight')
    else :
        # Set up the matplotlib figure
        f, ax = plt.subplots(figsize=(8, 5))
        f = sns.heatmap(data, cmap=cmap, vmax=1, center=0,
                square=True, linewidths=.5, cbar=False)
        f.figure.savefig(path, bbox_inches='tight')


# Generate a custom diverging colormap
cmap = sns.diverging_palette(h_neg=240, h_pos=0,s=75, l=50, n=1, as_cmap=True)

path = "plot/rq1-is-not-subset.png"

plot_heatmap(rq1, cmap, path)
metrics = {}
for a1 in attacks :
    m = {}
    for a2 in attacks :
        m[a2] = one_pov_relation(a1, a2)
    metrics[a1] = m  
    
one_bsem = pd.DataFrame(data=metrics)
# def plot_heatmap(data, cmap, path, annot=False) :
#     sns.set_theme(style="white")

#     # Draw the heatmap with the mask and correct aspect ratio
#     if annot :
#         f, ax = plt.subplots(figsize=(12, 6))
#         f = sns.heatmap(data, cmap=cmap, vmax=1, center=0, annot=annot, fmt=".3f",
#                         linewidths=.5, cbar_kws={"shrink": .5})
#         f.figure.savefig(path, bbox_inches='tight')
#     else :
#         # Set up the matplotlib figure
#         f, ax = plt.subplots(figsize=(8, 5))
#         f = sns.heatmap(data, cmap=cmap, vmax=1, center=0,
#                 square=True, linewidths=.5, cbar=False)
#         f.figure.savefig(path, bbox_inches='tight')

def plot_heatmap(metrics, cmap, fpath, vmin, vmax, annot=True): 

    df = pd.DataFrame(data=metrics)

    plt.figure(figsize=(12,9))
    fig = sns.heatmap(df, cmap=cmap, vmin=vmin, vmax=vmax, annot=annot, fmt=".3f", linewidth=0.7)
#     fig.set(xlabel='Train', ylabel='Test')
    fig.figure.savefig(fpath, bbox_inches='tight')
    plt.show()
# Generate a custom diverging colormap
cmap = sns.diverging_palette(h_neg=240, h_pos=0,s=75, l=50, n=1, as_cmap=True)

path = "plot/rq2-one-bsem.png"

plot_heatmap(one_bsem, "binary", path, 0, 1)
metrics = {}
for a1 in attacks :
    m = {}
    for a2 in attacks :
        m[a2] = BSEM(a1, a2)
    metrics[a1] = m  
    
bsem = pd.DataFrame(data=metrics)
def plot_half_heatmap(data, cmap, path) :
    sns.set_theme(style="white")

    # Generate a mask for the upper triangle
    mask = np.triu(np.ones_like(data, dtype=bool))

    # Set up the matplotlib figure
    f, ax = plt.subplots(figsize=(12, 9))

    # Draw the heatmap with the mask and correct aspect ratio
    f = sns.heatmap(data, mask=mask, cmap=cmap, vmax=1, center=0,
                square=True, linewidths=.5, cbar=False, annot=True)
    f.figure.savefig(path, bbox_inches='tight')


# Generate a custom diverging colormap
cmap = sns.diverging_palette(h_neg=240, h_pos=0,s=75, l=50, n=1, as_cmap=True)

path = "plot/rq2-bsem.png"

plot_half_heatmap(bsem, cmap, path)
!pip install scipy
1-bsem
import scipy.cluster.hierarchy as hcluster
linkage = hcluster.linkage(1 - bsem)
dendro  = hcluster.dendrogram(linkage, labels=bsem.columns, orientation="right")
import networkx as nx
G = nx.Graph()

for a1 in attacks: 
    for a2 in attacks :
        G.add_edge(a1, a2, weight=bsem[a1][a2])

pos = nx.spring_layout(G)  # default to scale=1
nx.draw(G, pos)
nx.draw(G, pos)
nx.draw_networkx_labels(G, pos, font_size=10)
plt.show() 
import numpy as np
a = np.zeros((50000, 3, 32, 32))
b = np.zeros((50000, 3, 32, 32))
np.concatenate((a, b)).shape
attacks = ["autoattack", "autopgd", "bim", "cw", "fgsm", "pgd", "deepfool", "newtonfool", "squareattack", "pixelattack", "spatialtransformation"]

cluster = ["autoattack", "autopgd", "bim", "cw", "pgd", "deepfool", "newtonfool"]
# cluster = ["autoattack", "autopgd", "bim", "pgd"]

def get_centroid(cluster, sim) :
    val = {}
    for c in cluster :
        val[c] = 0
        for c2 in cluster :
            val[c] += sim[c][c2]
    max_dist = val[cluster[0]]
    idx = cluster[0]
    for c in cluster :
        if val[c] > max_dist :
            max_dist = val[c]
            idx = c
    return idx
            
get_centroid(cluster, bsem)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1635395537525355, 'fraction_numerical': 0.022227856659905343, 'mean_word_length': 3.3858413639733134, 'pattern_counts': {'":': 4, '<': 6, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '631681', 'license': 'Apache-2.0', 'n_tokens_mistral': 4937, 'n_tokens_neox': 4724, 'n_words': 1054, 'path': 'cifar10/evaluation-metric-fine-grain.ipynb', 'repo_name': 'mhilmiasyrofi/Bag-of-Tricks-for-AT', 'size': '11845'}