<jupyter_start><jupyter_text>Automatic ExperimentsIn this notebook, we run bunch of experiments to assess the performance of stability training to improve robustness against Fast Sign Gradient perturbations.<jupyter_code>import sys, os 
sys.path.append(os.getcwd()+'/../src')
sys.path.append(os.getcwd()+'/../data')

import numpy
import matplotlib.pyplot as plt
%matplotlib inline
from IPython.display import clear_output

import architectures
import cifar10
import utils
import perturbations
X_test, y_test_lab = cifar10.load_cifar(dataset='testing', path=os.getcwd()+'/../data/cifar')
X_test = numpy.reshape(X_test, [X_test.shape[0], -1])
X_test = X_test/255.
X_test = X_test.astype(numpy.float32)
y_test = utils.labels_to_categoricals(y_test_lab)
if False:
    blank_net = architectures.CifarNet(path_to_logs=os.getcwd()+'/Logs')
    blank_net.load('Networks/BlankNet')
    blank_gradient = perturbations.compute_fs_grad_sym(blank_net, X_test, y_test)
    numpy.save('blankgradient', blank_gradient)
else:
    blank_gradient = numpy.load('blankgradient.npy')
class_net = architectures.CifarNet(path_to_logs=os.getcwd()+'/Logs')
class_net.load('Networks/ClassiclyTrained')
if False:
    class_gradient = perturbations.compute_fs_grad_sym(class_net, X_test, y_test)
    numpy.save('classgradient', class_gradient)
else:
    class_gradient = numpy.load('classgradient.npy')
stab_nets_name = ['Networks/StabilityTrained1', 'Networks/StabilityTrained2', 'Networks/StabilityTrained3']
stab_grad_files = ['stabgradient1.npy', 'stabgradient2.npy', 'stabgradient3.npy']
pre_rec_values = [0, 0.01, 0.03, 0.1]
for net_iter in range(0,1):
    stab_net = architectures.CifarNet(path_to_logs=os.getcwd()+'/Logs')
    stab_net.load(stab_nets_name[net_iter])
    stab_gradient = numpy.load(stab_grad_files[net_iter])
    
    ## Experiment1: Accuracy curve for recomputed perturbation
    epsilons = numpy.linspace(0.0001, 0.1, 20)
    accuracies = numpy.zeros([2,epsilons.shape[0]])
    for epsiter in range(0,epsilons.shape[0]):
        X_stab = X_test + epsilons[epsiter]*numpy.sign(stab_gradient)
        X_stab /= X_stab.max()
        X_class = X_test + epsilons[epsiter]*numpy.sign(class_gradient)
        X_class /= X_class.max()
        accuracies[0,epsiter] = class_net.test(X_class, y_test)
        accuracies[1,epsiter] = stab_net.test(X_stab, y_test)
    plt.plot(epsilons,accuracies[0], label='Classicly Trained')
    plt.plot(epsilons,accuracies[1], label='Stability Trained')
    plt.title("Accuracy for varying eps and recomputed perturbation (StabNet %i)"%(net_iter+1))
    plt.legend()
    plt.savefig('Figures/exp%i-1.pdf'%(net_iter+1))
    plt.show()

    ## Experiment2: Accuracy curve for blank perturbation
    epsilons = numpy.linspace(0.0001, 0.1, 20)
    accuracies = numpy.zeros([2,epsilons.shape[0]])
    for epsiter in range(0,epsilons.shape[0]):
        X_stab = X_test + epsilons[epsiter]*numpy.sign(blank_gradient)
        X_stab /= X_stab.max()
        accuracies[0,epsiter] = class_net.test(X_stab, y_test)
        accuracies[1,epsiter] = stab_net.test(X_stab, y_test)
    plt.plot(epsilons,accuracies[0], label='Classicly Trained')
    plt.plot(epsilons,accuracies[1], label='Stability Trained')
    plt.title("Accuracy for varying eps and blank perturbation (StabNet %i)"%(net_iter+1))
    plt.legend()
    plt.savefig('Figures/exp%i-2.pdf'%(net_iter+1))
    plt.show()
    
     ## Experiment3: Precision Recall curves for recomputed perturbations
    for pr_iter in range(0,4):
        top_ns = numpy.arange(1,11)
        accuracies = numpy.zeros([2, top_ns.shape[0]])
        eps = pre_rec_values[pr_iter]
        X_stab = X_test + eps*numpy.sign(stab_gradient)
        X_stab /= X_stab.max()
        X_class = X_test + eps*numpy.sign(class_gradient)
        X_class /= X_class.max()
        for itr in range(0,top_ns.shape[0]):
            accuracies[0,itr] = class_net.test(X_class, y_test, top=top_ns[itr])
            accuracies[1,itr] = stab_net.test(X_stab, y_test, top=top_ns[itr])
        plt.plot(top_ns,accuracies[0], label='Classicly Trained')
        plt.plot(top_ns,accuracies[1], label='Stability Trained')
        plt.title("Precision-Recall for eps=%f with recomputed perturbation (StabNet %i)"%(eps, net_iter+1))
        plt.legend(loc=4)
        plt.savefig('Figures/exp%i-3-%i.pdf'%(net_iter+1, pr_iter))
        plt.show()
        
    ## Experiment4: Precision Recall curves for blank perturbation
    for pr_iter in range(0,4):
        top_ns = numpy.arange(1,11)
        accuracies = numpy.zeros([2, top_ns.shape[0]])
        eps = pre_rec_values[pr_iter]
        X_stab = X_test + eps*numpy.sign(blank_gradient)
        X_stab /= X_stab.max()
        for itr in range(0,top_ns.shape[0]):
            accuracies[0,itr] = class_net.test(X_stab, y_test, top=top_ns[itr])
            accuracies[1,itr] = stab_net.test(X_stab, y_test, top=top_ns[itr])
        plt.plot(top_ns,accuracies[0], label='Classicly Trained')
        plt.plot(top_ns,accuracies[1], label='Stability Trained')
        plt.title("Precision-Recall for eps=%f with blank perturbation (StabNet %i)"%(eps, net_iter+1))
        plt.legend(loc=4)
        plt.savefig('Figures/exp%i-4-%i.pdf'%(net_iter+1, pr_iter))
        plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1370616113744076, 'fraction_numerical': 0.02009478672985782, 'mean_word_length': 4.1423001949317735, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '371308', 'license': 'MIT', 'n_tokens_mistral': 2056, 'n_tokens_neox': 1939, 'n_words': 320, 'path': 'notebooks/3_automatic_experiments.ipynb', 'repo_name': 'aPere3/MVAProject-RecVis16', 'size': '5275'}