<jupyter_start><jupyter_text>$A(M, X_B)$ = robust_accuracy$A(M_B, X_B)$ = trained_robust_accuracy$A(M_B, X_C)$ = cross_robust_accuracy  RQ1 - Motivating workWe hypothesize that each adversarial attack has a unique characteristic, thus the test cases generated from each adversarial attack are not targeting the same bug. To prove if the accuracy is not enough in measuring the performance of adversarial attacks, we need to demonstrate that $bugs_{X_C} \nsubseteq bugs_{X_B}$ and $bugs_{X_B} \nsubseteq bugs_{X_C}$. In the other word, there exist at least 2 adversarial examples $X_B$ and $X_C$ such that $A(M_C, X_B) < A(M_B, X_B)$ and $A(M_B, X_C) < A(M_C, X_C)$.<jupyter_code>def compare(a1, a2):
    trained_robust_acc = get_trained_robust_accuracy(a1) 
    cross_robust_acc = get_cross_robust_accuracy(a2, a1)
#     print("Trained Robust Accuracy: ", trained_robust_acc)
#     print("Cross Robust Accuracy: ", cross_robust_acc)
    return trained_robust_acc - cross_robust_acc
def get_comparison_metrics(attacks) :
    metrics = {}
    for a1 in attacks :
        m = {}
        for a2 in attacks :
            m[a2] = compare(a1, a2)
        metrics[a1] = m  
    return metrics
    
def plot_heatmap(metrics, cmap, fpath, vmin, vmax, annot=True): 

    df = pd.DataFrame(data=metrics)

    plt.figure(figsize=(12,9))
    fig = sns.heatmap(df, cmap=cmap, vmin=vmin, vmax=vmax, annot=annot, fmt=".3f", linewidth=0.7)
#     fig.set(xlabel='Train', ylabel='Test')
    fig.figure.savefig(fpath, bbox_inches='tight')
    plt.show()
metrics = get_comparison_metrics(attacks)
plot_heatmap(metrics, "coolwarm", "plot/rq1.png", vmin=-0.13, vmax=0.13, annot=True)<jupyter_output><empty_output><jupyter_text>$A(M_B, X_C)  0$. In the heatmap above, the value represent the difference between $A(M_C, X_C)$ and $A(M_B, X_C)$. We can see that several heatmaps have the red-like color which prove that this happen.For an example, if we take SpatialTransformation and PixelAttack then we get    * $A(M'_{SpatialTransformation}, X'_{PixelAttack}) - A(M'_{PixelAttack}, X'_{PixelAttack})$ = 0.65* $A(M'_{PixelAttack}, X'_{SpatialTranformation}) - A(M'_{SpatialTranformation}, X'_{SpatialTranformation})$ = 0.53It proves that SpatialTransformation has test cases that are attacking different bugs from any test cases in PixelAttack and vice versa. From this heatmap, we get 26 pairs of adversarial attacks that satisfy $A(M'_2, X'_2) - A(M'_1, X'_2) > 0$ and $A(M'_1, X'_1) - A(M'_2, X'_1) > 0$.<jupyter_code># Make Sure
train = "spatialtransformation"
test = "pixelattack"

trained_robust_accuracy = get_trained_robust_accuracy(train)
cross_robust_accuracy = get_cross_robust_accuracy(test, train)

print("Train Data: ", train)
print("Test Data: ", test)
print("=================")
print("Trained Robust Accuracy: ", trained_robust_accuracy)
print("Cross Robust Accuracy: ", cross_robust_accuracy)<jupyter_output>Train Data:  spatialtransformation
Test Data:  pixelattack
=================
Trained Robust Accuracy:  0.8617
Cross Robust Accuracy:  0.1<jupyter_text>$A(M'_{SpatialTransformation}, X'_{PixelAttack}) - A(M'_{PixelAttack}, X'_{PixelAttack})$ = 0.7551 - 0.1 = 0.65<jupyter_code>indexs =[]
attack_types = []
attack_the_same_bugs = "Attack The Same Bug"
attack_the_different_bugs = "Attack The DIfferent Bug"
proof = set()
for a1 in attacks : 
    for a2 in attacks :
        if a1 != a2 :
            index = None
            if a1 > a2 :
                index = a1 + "-" + a2
            else :
                index = a2 + "-" + a1

            indexs.append(index)
            if metrics[a1][a2] > 0 and metrics[a2][a1] > 0 :
                proof.add(index)
                attack_types.append(attack_the_different_bugs)
            else :
                attack_types.append(attack_the_same_bugs)
            
# len(proof)
# proof
df_attack_types = pd.DataFrame(data={"index": indexs, "attack_types": attack_types})

df_attack_types = df_attack_types.drop_duplicates().reset_index(drop=True)
df_attack_types.head()
df_attack_types.groupby("attack_types").count()
plt.figure(figsize=(10,5))
fig = sns.countplot(data=df_attack_types, x="attack_types")
fpath = "plot/rq1-histogram.png"
fig.figure.savefig(fpath)
plt.show()
different_bugs_metrics = {}


for a1 in attacks : 
    sm = {}
    for a2 in attacks :
        id = a1 + "-" + a2 
        if a1 + "-" + a2  in proof or a2 + "-" + a1 in proof :
            sm[a2] = 1
        else :
            sm[a2] = 0
    different_bugs_metrics[a1] = sm
    
different_bugs_metrics = pd.DataFrame(data=different_bugs_metrics)
def plot_half_heatmap(data, cmap, path) :
    sns.set_theme(style="white")

    # Generate a mask for the upper triangle
    mask = np.triu(np.ones_like(different_bugs_metrics, dtype=bool))

    # Set up the matplotlib figure
    f, ax = plt.subplots(figsize=(8, 5))

    # Draw the heatmap with the mask and correct aspect ratio
    f = sns.heatmap(data, mask=mask, cmap=cmap, vmax=1, center=0,
                square=True, linewidths=.5, cbar=False)
    f.figure.savefig(path, bbox_inches='tight')

def plot_heatmap(data, cmap, path, annot=False) :
    sns.set_theme(style="white")

    # Draw the heatmap with the mask and correct aspect ratio
    if annot :
        f, ax = plt.subplots(figsize=(12, 6))
        f = sns.heatmap(data, cmap=cmap, vmax=1, center=0, annot=annot, fmt=".3f",
                        linewidths=.5, cbar_kws={"shrink": .5})
        f.figure.savefig(path, bbox_inches='tight')
    else :
        # Set up the matplotlib figure
        f, ax = plt.subplots(figsize=(8, 5))
        f = sns.heatmap(data, cmap=cmap, vmax=1, center=0,
                square=True, linewidths=.5, cbar=False)
        f.figure.savefig(path, bbox_inches='tight')


# Generate a custom diverging colormap
cmap = sns.diverging_palette(h_neg=240, h_pos=0,s=75, l=50, n=1, as_cmap=True)

path = "plot/rq1-attack-diffferent-bugs.png"

plot_half_heatmap(different_bugs_metrics, cmap, path)<jupyter_output><empty_output><jupyter_text>In the contrary, the AutoAttack, AutoPGD, BIM, CW , FGSM, and PGD, both have a 0 value. The test cases generated from those adversarial attacks don't attack the different bugs. we know that $bugs_{X_B} \subset bugs_{X_C}$ is satisfied if $A(M_C, X_B) > A(M_B, X_B)$ and $A(M_B, X_C) < A(M_C, X_C)$.<jupyter_code>subset_bugs_metrics = {}

for a1 in attacks : 
    sm = {}
    for a2 in attacks :
        id = a1 + "-" + a2 
        if metrics[a1][a2] < 0 and metrics[a2][a1] > 0 :
            sm[a2] = -1
        else :
            sm[a2] = 0
    subset_bugs_metrics[a1] = sm

subset_bugs_metrics = pd.DataFrame(data=subset_bugs_metrics)
    
# Generate a custom diverging colormap
cmap = sns.diverging_palette(h_neg=240, h_pos=0,s=75, l=50, n=1, as_cmap=True)

path = "plot/rq1-subset-bugs.png"

plot_heatmap(subset_bugs_metrics, cmap, path)<jupyter_output><empty_output><jupyter_text>In the Figure above, the horisontal axis means the first term used and the vertical axis is the second term used. For example, from the above metric, $bugs_{X_{AutoPGD}} \subset bugs_{X_{AutoAttack}}$, $bugs_{X_{FGSM}} \subset bugs_{X_{PGD}}$, $bugs_{X_{DeepFool}} \subset bugs_{X_{NewtonFool}}$, etc.  RQ2 - How to use BSEM? How BSEM compared to the existing metric?Given a model $M$, an original data $X$, a list of $n$ adversarial attack $AA = \{ aa_1, aa_2, aa_3, ... , aa_n \}$, an adversarial defense technique $AD$. First, we generate adversarial examples using each adversarial attack from $AA$. For each pair of adversarial examples generated from different adversarial attack, we measure BSEM. Then we make a leaderboard that mimic the existing evaluation relative to the adversarial defense $AD$.  One Way Relation<jupyter_code>def one_pov_relation(a1, a2):
    robust_acc = get_robust_accuracy(a2)
    trained_robust_acc = get_trained_robust_accuracy(a2)
    cross_robust_acc = get_cross_robust_accuracy(a1, a2)
#     print("RobustAccuracy(original, {}): {:.4f}".format(a2, robust_acc))
#     print("RobustAccuracy({}, {}): {:.4f}".format(a2, a2, trained_robust_acc))
#     print("RobustAccuracy({}, {}): {:.4f}".format(a1, a2, cross_robust_acc))
    return (min(cross_robust_acc, trained_robust_acc) - robust_acc) / (trained_robust_acc - robust_acc)


def measure_relation(a1, a2) :
#     print((one_pov_relation(a1, a2) + one_pov_relation(a2, a1))/2)
#     return max(0, (one_pov_relation(a1, a2) + one_pov_relation(a2, a1))/2)
    return max(0, one_pov_relation(a1, a2))

# measure_relation("spatialtransformation", "ffgsm")
measure_relation("pgd", "autoattack")
measure_relation("squareattack", "autoattack")
measure_relation("autoattack", "squareattack")
#TODO: 
# cek formula -> what does it means?
# benerin angka dibelakakgn koma
# owr: one way relation
owr = {}
for a1 in attacks :
    m = {}
    for a2 in attacks :
        m[a2] = measure_relation(a1, a2)
    owr[a1] = m

owr = pd.DataFrame(data=owr)


# Generate a custom diverging colormap
cmap = sns.diverging_palette(h_neg=300, h_pos=12.2,s=75, l=37, n=1, as_cmap=True)

path = "plot/rq2-bsem.png"

plot_heatmap(owr, cmap, path, annot=True)
leaderboard = df[["attack","robust_accuracy"]]
leaderboard.sort_values(["robust_accuracy"]).reset_index(drop=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13047238136417685, 'fraction_numerical': 0.021727380823694735, 'mean_word_length': 4.100330760749724, 'pattern_counts': {'":': 3, '<': 24, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '509847', 'license': 'MIT', 'n_tokens_mistral': 3582, 'n_tokens_neox': 3330, 'n_words': 815, 'path': 'oat-evaluation-metric.ipynb', 'repo_name': 'mhilmiasyrofi/Once-for-All-Adversarial-Training', 'size': '9251'}