<jupyter_start><jupyter_text>UAP Generate & Evaluate (whitebox)attacks- sgd-uap (untargeted) eps = 2-12dataset- cifar10- svhnmodel_name- resnet18- [soft-pruning]    - resnet18_sfP    - resnet18_sfP-mixup    - resnet18_sfP-cutout- [post-training pruning]    - resnet18_PX_0.Y        - X = 2, 3, 4        - Y = 3, 6, 9<jupyter_code>import numpy as np
import pandas as pd
import os
import torch
import json

from utils_attack import uap_batch
from utils_model import *

dir_results = './results/'
dir_uap = './uaps/'
nb_epoch = 10
atk_name = 'sgd-uap'
y_target = -1
model_names =  ['resnet18', 'resnet18_sfP', 'resnet18_sfP-mixup', 'resnet18_sfP-cutout']
for layer in [2, 3, 4]:
    for prune_pct in ['0.3', '0.6', '0.9']:
        model_names.append('resnet18_P%i-%s' % (layer, prune_pct))
model_names
def eval_uap(uap):    
    top, top_probs, top1acc, top5acc, outputs, labels = evaluate(model, testloader, uap = uap)

    print('Top 1 accuracy', sum(top1acc) / len(labels))
    print('Top 5 accuracy', sum(top5acc) / len(labels))

    dict_entry = {}
    dict_entry['model_name'] = model_name
    dict_entry['dataset'] = dataset

    dict_entry['atk_name'] = atk_name
    dict_entry['atk_param'] = y_target
    dict_entry['eps'] = eps
    if atk_name[:3] == 'sgd': dict_entry['nb_epoch'] = nb_epoch

    dict_entry['top1acc'] = sum(top1acc) / len(labels)
    dict_entry['top5acc'] = sum(top5acc) / len(labels)
    dict_entry['UER'] = 1 - sum(top1acc) / len(labels)
    print('UER ', dict_entry['UER'])
    print('\n')
    if y_target >= 0: dict_entry['tgt_success'] = sum(outputs == y_target) / len(labels)

    # Output distribution
    for i in range(10):
        dict_entry['label_dist%i' % i] = sum(outputs == i) / len(labels)

    all_results.append(dict_entry)<jupyter_output><empty_output><jupyter_text>all_results = []for dataset in ['cifar10']:    testloader = get_testdata(dataset, batch_size)        for model_name in model_names:        model = get_model(model_name, dataset, batch_size)        for eps in range(2, 13, 2):                         GENERATE UAP                         Save directory            uap_save_path = dir_uap + dataset + '/' + model_name            if not os.path.exists(uap_save_path):                os.mkdir(uap_save_path)                             Compute UAP            print(uap_save_path + '/sgd-eps%i.pth' % eps)            uap, losses = uap_batch(model, testloader, nb_epoch, eps / 255, beta = 8, y_target = y_target)                        torch.save(uap, uap_save_path + '/sgd-eps%i.pth' % eps)                         EVALUATE UAP            eval_uap(uap)                    pd.DataFrame(all_results).to_csv(dir_results + 'eval_whitebox-%s.csv' % dataset)  GENERATE UAP since masking operation in post-training pruning has issues 1627 too big for SGD-UAP computation (can't fit GPU memory)dataset = 'svhn'batch_size = 500testloader = get_testdata(dataset, batch_size)for model_name in model_names:    model = get_model(model_name, dataset, batch_size)    for eps in range(2, 13, 2):         Save directory        uap_save_path = dir_uap + dataset + '/' + model_name        if not os.path.exists(uap_save_path):            os.mkdir(uap_save_path)         Compute UAP        print(uap_save_path + '/sgd-eps%i.pth' % eps)        uap, losses = uap_batch(model, testloader, nb_epoch, eps / 255, beta = 8, y_target = y_target, cut_end = True)        torch.save(uap, uap_save_path + '/sgd-eps%i.pth' % eps)<jupyter_code>all_results = []
dataset = 'svhn'
batch_size = 1627

testloader = get_testdata(dataset, batch_size)

for model_name in model_names:
    model = get_model(model_name, dataset, batch_size)
    model.eval()

    for eps in range(2, 13, 2):
        uap_save_path = dir_uap + dataset + '/' + model_name
        print(uap_save_path + '/sgd-eps%i.pth' % eps)
        uap = torch.load(uap_save_path + '/sgd-eps%i.pth' % eps)
        eval_uap(uap)

    pd.DataFrame(all_results).to_csv(dir_results + 'eval_whitebox-%s.csv' % dataset)<jupyter_output>Using downloaded and verified file: ./data/svhn/test_32x32.mat
Loading svhn/resnet18.pth
./uaps/svhn/resnet18/sgd-eps2.pth
 Completed: 0.00%
Top 1 accuracy 0.9303549477566072
Top 5 accuracy 0.9934311616472035
UER  0.06964505224339279


./uaps/svhn/resnet18/sgd-eps4.pth
 Completed: 0.00%
Top 1 accuracy 0.8367393976644131
Top 5 accuracy 0.9795251997541488
UER  0.16326060233558692


./uaps/svhn/resnet18/sgd-eps6.pth
 Completed: 0.00%
Top 1 accuracy 0.667178856791641
Top 5 accuracy 0.9275507068223725
UER  0.332821143208359


./uaps/svhn/resnet18/sgd-eps8.pth
 Completed: 0.00%
Top 1 accuracy 0.5442148125384143
Top 5 accuracy 0.8760755992624463
UER  0.4557851874615857


./uaps/svhn/resnet18/sgd-eps10.pth
 Completed: 0.00%
Top 1 accuracy 0.41037953288260604
Top 5 accuracy 0.7894130301167793
UER  0.5896204671173939


./uaps/svhn/resnet18/sgd-eps12.pth
 Completed: 0.00%
Top 1 accuracy 0.3676628764597419
Top 5 accuracy 0.7935233558696988
UER  0.6323371235402582


Loading svhn/resnet18_sfP.pth
Fi[...]<jupyter_text>View Evaluation Results<jupyter_code>df = pd.read_csv(dir_results + 'eval_whitebox-svhn.csv', index_col = 0)
df[(df['dataset'] == 'svhn') & (df['eps'] == 8)].sort_values(by = 'UER')
df = pd.read_csv(dir_results + 'eval_whitebox-cifar10.csv', index_col = 0)
df[(df['dataset'] == 'cifar10') & (df['eps'] == 8)].sort_values(by = 'UER')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1266480965645311, 'fraction_numerical': 0.08653667595171774, 'mean_word_length': 3.393148450244698, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '47572', 'license': 'MIT', 'n_tokens_mistral': 2373, 'n_tokens_neox': 2073, 'n_words': 421, 'path': '11-02 eval_untgt-whitebox.ipynb', 'repo_name': 'mataach/adversarial_ml', 'size': '5385'}