<jupyter_start><jupyter_text>Preliminary CIFAR-10 AnalysisSimple example using CIFAR-10 and a simple fast gradient sign attack.<jupyter_code>%matplotlib inline
%load_ext autoreload
%autoreload 2

import os

import numpy as np
from scipy.stats import ks_2samp
import pylab as plt
import pandas as pd

from sklearn import svm
from sklearn.metrics import roc_curve, auc


from functools import reduce
## Load data set and show some related information
df = pd.read_pickle('../cifar10_stats_df.pkl')

print(df.dtypes.index)

datasets = pd.unique(df['Dataset'])
examples = pd.unique(df['Example#'])

print('frame has %d datasets and %d unique examples' % (len(datasets), len(examples)))
print(datasets)
## Here we examine whether the gradient norm seems to be indicative
## of AE vs clean examples.

# Extract the sub-table corresponding only to the gradient-related rows.
df_grad_only = df[df['direction_type'] == 'gradient']

# Show distribution of gradient norms
gradient_norms = []
for idx, dset in enumerate(datasets):
    rows = (df_grad_only['Dataset'] == dset)
    n = np.sum(rows)
    print('%s has %d examples' % (dset, n))
    
    gradient_norms.append(df_grad_only['ell2_grad'][rows])

## visually inspect the distributions
plt.figure(figsize=(10,5))
plt.boxplot(gradient_norms, labels=datasets)
plt.plot()

#plt.figure()
#for ii in range(len(gradient_norms)):
#    plt.hist(np.log(gradient_norms[ii]), alpha=0.5, label=datasets[ii])
#plt.legend(loc='upper left')
#plt.show()


## Compare each AE distribution to the clean example distribution with a simple statistical test.

for ii in range(len(datasets)):
    for jj in range(ii+1,len(datasets)):
        comparison = [gradient_norms[ii], gradient_norms[jj]]
        _, pv = ks_2samp(*comparison)
        print('"%s" vs "%s" : pvalue=%0.3g' % (datasets[ii], datasets[jj], pv))

# look now at the per-example differences between ||J|| for clean vs AE.

all_pairwise_diffs = []
ae_datasets = [x for x in datasets if x != 'cifar10']


df0 = df_grad_only[ df_grad_only['Dataset'] == 'cifar10' ]
df0.sort_values('Example#')


for ds_name in ae_datasets:
    
    print(ds_name)
    df1 = df_grad_only[ df_grad_only['Dataset'] == ds_name ]
    df1.sort_values('Example#')
    print(df1.shape)

    in_both = df0['Example#'].isin(df1['Example#'])
    assert(np.sum(in_both) == df1.shape[0])
    
    delta = np.array(df1['ell2_grad']) - np.array(df0['ell2_grad'][in_both])
    all_pairwise_diffs.append(delta)
    
    
       
plt.figure(figsize=(10,5))
plt.boxplot(all_pairwise_diffs, labels=ae_datasets)
plt.show()
# consider the "classification" performance if we use a single feature, the gradient norm.
print(datasets == 'cifar10')
idx_0 = np.nonzero(datasets == 'cifar10')[0][0]

for idx_1 in range(len(datasets)):
    if idx_1 == idx_0:
        continue
    
    class_0 = gradient_norms[idx_0]
    class_1 = gradient_norms[idx_1]
    
    y_score = np.concatenate([class_0, class_1], axis=0)
    y_true = np.concatenate([np.zeros((len(class_0),)), np.ones((len(class_1),))])
    
    fpr, tpr, _ = roc_curve(y_true, y_score)
    auc(fpr, tpr)

    plt.figure()
    plt.plot(fpr, tpr, color='darkorange')
    plt.plot([0,1], [0,1], color='navy', linestyle='--')
    plt.title('%s vs %s' % (datasets[idx_0], datasets[idx_1]))
    plt.xlabel('fpr');  plt.ylabel('tpr')
    plt.show()<jupyter_output>[ True False False False False False False False]
{'fraction_non_alphanumeric': 0.1426056338028169, 'fraction_numerical': 0.018485915492957746, 'mean_word_length': 4.489533011272142, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '526127', 'license': 'Apache-2.0', 'n_tokens_mistral': 1281, 'n_tokens_neox': 1230, 'n_words': 296, 'path': 'src/Analysis/prelim_cifar10.ipynb', 'repo_name': 'mjpekala/ae-stability-study', 'size': '3408'}