<jupyter_start><jupyter_text>Demo - Performance Comparison (CIFAR10)  1. Load CIAFR10<jupyter_code># https://github.com/RobustBench/robustbench
from robustbench.data import load_cifar10
from robustbench.utils import load_model, clean_accuracy
from utils import l2_distance

images, labels = load_cifar10(n_examples=50)
device = "cuda"<jupyter_output>Files already downloaded and verified<jupyter_text>2. Standard Accuracy<jupyter_code>model_list = ['Wong2020Fast', 'Rice2020Overfitting', 'Carmon2019Unlabeled']
for model_name in model_list:
    model = load_model(model_name, norm='Linf').to(device)
    acc = clean_accuracy(model, images.to(device), labels.to(device))
    print('Model: {}'.format(model_name))
    print('- Standard Acc: {}'.format(acc))<jupyter_output>Model: Wong2020Fast
- Standard Acc: 0.92
Model: Rice2020Overfitting
- Standard Acc: 0.86
Model: Carmon2019Unlabeled
- Standard Acc: 0.92<jupyter_text>3. Torchattacks, Foolbox and ART<jupyter_code>import datetime
import numpy as np
import warnings

warnings.filterwarnings(action='ignore')

import torch
import torch.nn as nn
import torch.optim as optim

# https://github.com/Harry24k/adversarial-attacks-pytorch
import torchattacks
print("torchattacks %s"%(torchattacks.__version__))

# https://github.com/bethgelab/foolbox
import foolbox as fb
print("foolbox %s"%(fb.__version__))

# https://github.com/IBM/adversarial-robustness-toolbox
import art
import art.attacks.evasion as evasion
from art.classifiers import PyTorchClassifier
print("art %s"%(art.__version__))<jupyter_output>torchattacks 2.13.3
foolbox 3.0.0
art 1.2.0<jupyter_text>3.1. Linf  FGSM<jupyter_code>for model_name in model_list:
    print('Model: {}'.format(model_name))
    model = load_model(model_name, norm='Linf').to(device)
    
    print("- Torchattacks")
    atk = torchattacks.FGSM(model, eps=8/255)
    start = datetime.datetime.now()
    adv_images = atk(images, labels)
    end = datetime.datetime.now()
    acc = clean_accuracy(model, adv_images, labels)
    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))
    
    print("- Foolbox")
    fmodel = fb.PyTorchModel(model, bounds=(0, 1))
    atk = fb.attacks.LinfFastGradientAttack(random_start=False)
    start = datetime.datetime.now()
    _, adv_images, _ = atk(fmodel, images.to('cuda:0'), labels.to('cuda:0'), epsilons=8/255)
    end = datetime.datetime.now()
    acc = clean_accuracy(model, adv_images, labels)
    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))
    
    print("- ART")
    classifier = PyTorchClassifier(model=model, clip_values=(0, 1),
                                   loss=nn.CrossEntropyLoss(),
                                   optimizer=optim.Adam(model.parameters(), lr=0.01),
                                   input_shape=(3, 32, 32), nb_classes=10)
    atk = evasion.FastGradientMethod(norm=np.inf, batch_size=50,
                                     classifier=classifier, eps=8/255)
    start = datetime.datetime.now()
    adv_images = torch.tensor(atk.generate(images, labels)).to(device)
    end = datetime.datetime.now()
    acc = clean_accuracy(model, adv_images, labels)
    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))
    
    print()<jupyter_output>Model: Wong2020Fast
- Torchattacks
- Robust Acc: 0.48 (25 ms)
- Foolbox
- Robust Acc: 0.48 (18 ms)
- ART
- Robust Acc: 0.48 (67 ms)

Model: Rice2020Overfitting
- Torchattacks
- Robust Acc: 0.62 (91 ms)
- Foolbox
- Robust Acc: 0.62 (55 ms)
- ART
- Robust Acc: 0.62 (733 ms)

Model: Carmon2019Unlabeled
- Torchattacks
- Robust Acc: 0.68 (16 ms)
- Foolbox
- Robust Acc: 0.68 (26 ms)
- ART
- Robust Acc: 0.68 (217 ms)<jupyter_text>BIM<jupyter_code>for model_name in model_list:
    print('Model: {}'.format(model_name))
    model = load_model(model_name, norm='Linf').to(device)
    
    print("- Torchattacks")
    atk = torchattacks.BIM(model, eps=8/255, alpha=2/255, steps=10)
    start = datetime.datetime.now()
    adv_images = atk(images, labels)
    end = datetime.datetime.now()
    acc = clean_accuracy(model, adv_images, labels)
    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))
    
    print("- Foolbox")
    fmodel = fb.PyTorchModel(model, bounds=(0, 1))
    atk = fb.attacks.LinfBasicIterativeAttack(abs_stepsize=2/255, steps=10, random_start=False)
    start = datetime.datetime.now()
    _, adv_images, _ = atk(fmodel, images.to('cuda:0'), labels.to('cuda:0'), epsilons=8/255)
    end = datetime.datetime.now()
    acc = clean_accuracy(model, adv_images, labels)
    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))
    
    print("- ART")
    classifier = PyTorchClassifier(model=model, clip_values=(0, 1),
                                   loss=nn.CrossEntropyLoss(),
                                   optimizer=optim.Adam(model.parameters(), lr=0.01),
                                   input_shape=(3, 32, 32), nb_classes=10)
    atk = evasion.BasicIterativeMethod(batch_size=50,
                                       classifier=classifier, eps=8/255,
                                       eps_step=2/255, max_iter=10)
    start = datetime.datetime.now()
    adv_images = torch.tensor(atk.generate(images, labels)).to(device)
    end = datetime.datetime.now()
    acc = clean_accuracy(model, adv_images, labels)
    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))
    
    print()<jupyter_output>Model: Wong2020Fast
- Torchattacks
- Robust Acc: 0.44 (170 ms)
- Foolbox
- Robust Acc: 0.44 (230 ms)
- ART
- Robust Acc: 0.44 (485 ms)

Model: Rice2020Overfitting
- Torchattacks
- Robust Acc: 0.58 (2798 ms)
- Foolbox
- Robust Acc: 0.58 (3331 ms)
- ART
- Robust Acc: 0.58 (4695 ms)

Model: Carmon2019Unlabeled
- Torchattacks
- Robust Acc: 0.58 (732 ms)
- Foolbox
- Robust Acc: 0.58 (917 ms)
- ART
- Robust Acc: 0.58 (1414 ms)<jupyter_text>PGD<jupyter_code>for model_name in model_list:
    print('Model: {}'.format(model_name))
    model = load_model(model_name, norm='Linf').to(device)
    
    print("- Torchattacks")
    atk = torchattacks.PGD(model, eps=8/255, alpha=2/255, steps=10, random_start=False)
    start = datetime.datetime.now()
    adv_images = atk(images, labels)
    end = datetime.datetime.now()
    acc = clean_accuracy(model, adv_images, labels)
    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))
    
    print("- Foolbox")
    fmodel = fb.PyTorchModel(model, bounds=(0, 1))
    atk = fb.attacks.LinfPGD(abs_stepsize=2/255, steps=10, random_start=False)
    start = datetime.datetime.now()
    _, adv_images, _ = atk(fmodel, images.to('cuda:0'), labels.to('cuda:0'), epsilons=8/255)
    end = datetime.datetime.now()
    acc = clean_accuracy(model, adv_images, labels)
    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))
    
    print("- ART")
    classifier = PyTorchClassifier(model=model, clip_values=(0, 1),
                                   loss=nn.CrossEntropyLoss(),
                                   optimizer=optim.Adam(model.parameters(), lr=0.01),
                                   input_shape=(3, 32, 32), nb_classes=10)
    atk = evasion.ProjectedGradientDescent(batch_size=50, num_random_init=0,
                                           classifier=classifier, eps=8/255,
                                           eps_step=2/255, max_iter=10)
    start = datetime.datetime.now()
    adv_images = torch.tensor(atk.generate(images, labels)).to(device)
    end = datetime.datetime.now()
    acc = clean_accuracy(model, adv_images, labels)
    print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))
    
    print()<jupyter_output>Model: Wong2020Fast
- Torchattacks
- Robust Acc: 0.44 (140 ms)
- Foolbox
- Robust Acc: 0.44 (237 ms)
- ART
- Robust Acc: 0.44 (480 ms)

Model: Rice2020Overfitting
- Torchattacks
- Robust Acc: 0.58 (2697 ms)
- Foolbox
- Robust Acc: 0.58 (3353 ms)
- ART
- Robust Acc: 0.58 (4720 ms)

Model: Carmon2019Unlabeled
- Torchattacks
- Robust Acc: 0.58 (682 ms)
- Foolbox
- Robust Acc: 0.58 (920 ms)
- ART
- Robust Acc: 0.58 (1426 ms)<jupyter_text>3.2. L2  DeepFool<jupyter_code>for model_name in model_list:
    print('Model: {}'.format(model_name))
    model = load_model(model_name, norm='Linf').to(device)
    
    print("- Torchattacks")
    atk = torchattacks.DeepFool(model, steps=50, overshoot=0.02)
    start = datetime.datetime.now()
    adv_images = atk(images, labels)
    end = datetime.datetime.now()
    acc = clean_accuracy(model, adv_images, labels)
    l2 = l2_distance(model, images, adv_images, labels, device=device)
    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,
                                                         int((end-start).total_seconds()*1000)))
    
    print("- Foolbox")
    fmodel = fb.PyTorchModel(model, bounds=(0, 1))
    atk = fb.attacks.L2DeepFoolAttack(steps=50, candidates=10, overshoot=0.02)
    start = datetime.datetime.now()
    _, adv_images, _ = atk(fmodel, images.to('cuda'), labels.to('cuda'), epsilons=1)
    end = datetime.datetime.now()
    acc = clean_accuracy(model, adv_images, labels)
    l2 = l2_distance(model, images, adv_images, labels, device=device)
    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,
                                                         int((end-start).total_seconds()*1000)))

    print("- ART")
    classifier = PyTorchClassifier(model=model, clip_values=(0, 1),
                                   loss=nn.CrossEntropyLoss(),
                                   optimizer=optim.Adam(model.parameters(), lr=0.01),
                                   input_shape=(3, 32, 32), nb_classes=10)
    atk = evasion.DeepFool(classifier=classifier, max_iter=50,
                           batch_size=50)

    start = datetime.datetime.now()
    adv_images = torch.tensor(atk.generate(images, labels)).to(device)
    end = datetime.datetime.now()
    acc = clean_accuracy(model, adv_images, labels)
    l2 = l2_distance(model, images, adv_images, labels, device=device)
    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,
                                                         int((end-start).total_seconds()*1000)))
    
    print()<jupyter_output>Model: Wong2020Fast
- Torchattacks
- Robust Acc: 0.0 / L2: 1.0 (9952 ms)
- Foolbox
- Robust Acc: 0.4 / L2: 0.41 (1992 ms)
- ART
- Robust Acc: 0.04 / L2: 1.1 (18128 ms)

Model: Rice2020Overfitting
- Torchattacks
- Robust Acc: 0.0 / L2: 1.2 (39131 ms)
- Foolbox
- Robust Acc: 0.36 / L2: 0.47 (20359 ms)
- ART
- Robust Acc: 0.1 / L2: 1.3 (29016 ms)

Model: Carmon2019Unlabeled
- Torchattacks
- Robust Acc: 0.0 / L2: 1.8 (11555 ms)
- Foolbox
- Robust Acc: 0.46 / L2: 0.47 (5996 ms)
- ART
- Robust Acc: 0.06 / L2: 1.9 (8667 ms)<jupyter_text>CW<jupyter_code>for model_name in model_list:
    print('Model: {}'.format(model_name))
    model = load_model(model_name, norm='Linf').to(device)
    
    print("- Torchattacks")
    atk = torchattacks.CW(model, c=1, kappa=0, steps=100, lr=0.01)
    start = datetime.datetime.now()
    adv_images = atk(images, labels)
    end = datetime.datetime.now()
    acc = clean_accuracy(model, adv_images, labels)
    l2 = l2_distance(model, images, adv_images, labels, device=device)
    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,
                                                         int((end-start).total_seconds()*1000)))
    
    print("- Foolbox")
    fmodel = fb.PyTorchModel(model, bounds=(0, 1))
    atk = fb.attacks.L2CarliniWagnerAttack(binary_search_steps=1, initial_const=1,
                                           confidence=0, steps=100, stepsize=0.01)
    start = datetime.datetime.now()
    _, adv_images, _ = atk(fmodel, images.to('cuda'), labels.to('cuda'), epsilons=1)
    end = datetime.datetime.now()
    acc = clean_accuracy(model, adv_images, labels)
    l2 = l2_distance(model, images, adv_images, labels, device=device)
    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,
                                                         int((end-start).total_seconds()*1000)))

    print("- ART")
    classifier = PyTorchClassifier(model=model, clip_values=(0, 1),
                                   loss=nn.CrossEntropyLoss(),
                                   optimizer=optim.Adam(model.parameters(), lr=0.01),
                                   input_shape=(3, 32, 32), nb_classes=10)
    atk = evasion.CarliniL2Method(batch_size=50, classifier=classifier, 
                                  binary_search_steps=1, initial_const=1,
                                  confidence=0, max_iter=100,
                                  learning_rate=0.01)
    start = datetime.datetime.now()
    adv_images = torch.tensor(atk.generate(images, labels)).to(device)
    end = datetime.datetime.now()
    acc = clean_accuracy(model, adv_images, labels)
    l2 = l2_distance(model, images, adv_images, labels, device=device)
    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,
                                                         int((end-start).total_seconds()*1000)))
    
    print()<jupyter_output>Model: Wong2020Fast
- Torchattacks
- Robust Acc: 0.14 / L2: 0.61 (4372 ms)
- Foolbox
- Robust Acc: 0.32 / L2: 0.41 (4550 ms)
- ART
- Robust Acc: 0.24 / L2: 0.67 (72051 ms)

Model: Rice2020Overfitting
- Torchattacks
- Robust Acc: 0.22 / L2: 0.56 (43656 ms)
- Foolbox
- Robust Acc: 0.34 / L2: 0.43 (44207 ms)
- ART
- Robust Acc: 0.26 / L2: 0.65 (705790 ms)

Model: Carmon2019Unlabeled
- Torchattacks
- Robust Acc: 0.26 / L2: 0.48 (13358 ms)
- Foolbox
- Robust Acc: 0.32 / L2: 0.42 (13433 ms)
- ART
- Robust Acc: 0.26 / L2: 0.62 (206078 ms)<jupyter_text>PGD L2<jupyter_code>for model_name in model_list:
    print('Model: {}'.format(model_name))
    model = load_model(model_name, norm='Linf').cuda()
    
    print("- Torchattacks")
    atk = torchattacks.PGDL2(model, eps=128/255, alpha=15/255, steps=10, random_start=False)
    start = datetime.datetime.now()
    adv_images = atk(images, labels)
    end = datetime.datetime.now()
    acc = clean_accuracy(model, adv_images, labels)
    l2 = l2_distance(model, images, adv_images, labels, device=device)
    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,
                                                         int((end-start).total_seconds()*1000)))
    
    print("- Foolbox")
    fmodel = fb.PyTorchModel(model, bounds=(0, 1))
    atk = fb.attacks.L2PGD(abs_stepsize=15/255, steps=10, random_start=False)
    start = datetime.datetime.now()
    _, adv_images, _ = atk(fmodel, images.to('cuda:0'), labels.to('cuda:0'), epsilons=128/255)
    end = datetime.datetime.now()
    acc = clean_accuracy(model, adv_images, labels)
    l2 = l2_distance(model, images, adv_images, labels, device=device)
    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,
                                                         int((end-start).total_seconds()*1000)))
    
    print("- ART")
    classifier = PyTorchClassifier(model=model, clip_values=(0, 1),
                                   loss=nn.CrossEntropyLoss(),
                                   optimizer=optim.Adam(model.parameters(), lr=0.01),
                                   input_shape=(3, 32, 32), nb_classes=10)
    atk = evasion.ProjectedGradientDescent(batch_size=50, num_random_init=0,
                                           norm = 2, classifier=classifier, eps=128/255,
                                           eps_step=15/255, max_iter=10)
    start = datetime.datetime.now()
    adv_images = torch.tensor(atk.generate(images, labels)).to(device)
    end = datetime.datetime.now()
    acc = clean_accuracy(model, adv_images, labels)
    l2 = l2_distance(model, images, adv_images, labels, device=device)
    print('- Robust Acc: {} / L2: {:1.2} ({} ms)'.format(acc, l2,
                                                         int((end-start).total_seconds()*1000)))
    
    print()<jupyter_output>Model: Wong2020Fast
- Torchattacks
- Robust Acc: 0.68 / L2: 0.5 (163 ms)
- Foolbox
- Robust Acc: 0.68 / L2: 0.5 (266 ms)
- ART
- Robust Acc: 0.68 / L2: 0.5 (479 ms)

Model: Rice2020Overfitting
- Torchattacks
- Robust Acc: 0.7 / L2: 0.5 (2801 ms)
- Foolbox
- Robust Acc: 0.7 / L2: 0.5 (3494 ms)
- ART
- Robust Acc: 0.7 / L2: 0.5 (4797 ms)

Model: Carmon2019Unlabeled
- Torchattacks
- Robust Acc: 0.68 / L2: 0.5 (719 ms)
- Foolbox
- Robust Acc: 0.68 / L2: 0.5 (959 ms)
- ART
- Robust Acc: 0.68 / L2: 0.5 (1424 ms)<jupyter_text>4. Torchattacks and AutoAttack<jupyter_code>from torchattacks.attack import Attack
import autoattack
class AutoAttack(Attack):
    def __init__(self, model, eps, norm, seed, version):
        super(AutoAttack, self).__init__("AutoAttack", model)
        self.adversary = autoattack.AutoAttack(self.model, norm=norm, seed=seed,
                                               eps=eps, version=version, verbose=False)
        self._attack_mode = 'only_default'

    def forward(self, images, labels):
        adv_images = self.adversary.run_standard_evaluation(images.cuda(), labels.cuda(),
                                                            bs=images.shape[0])
        return adv_images
model_list = ['Wong2020Fast']
versions = ['standard', 'plus', 'rand']
for model_name in model_list:
    print('Model: {}'.format(model_name))
    model = load_model(model_name, norm='Linf').to(device)
    
    for version in versions:
        print("- Torchattacks (%s)"%(version))
        atk = torchattacks.AutoAttack(model, norm='Linf', eps=8/255, seed=5, version=version)
        start = datetime.datetime.now()
        adv_images = atk(images, labels)
        end = datetime.datetime.now()
        acc = clean_accuracy(model, adv_images, labels)
        print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))

        print("- AutoAttack (%s)"%(version))
        atk = AutoAttack(model, norm='Linf', eps=8/255, seed=5, version=version)
        start = datetime.datetime.now()
        adv_images = atk(images, labels)
        end = datetime.datetime.now()
        acc = clean_accuracy(model, adv_images, labels)
        print('- Robust Acc: {} ({} ms)'.format(acc, int((end-start).total_seconds()*1000)))
    
    print()<jupyter_output>Model: Wong2020Fast
- Torchattacks (standard)
- Robust Acc: 0.4 (82904 ms)
- AutoAttack (standard)
- Robust Acc: 0.4 (103209 ms)
- Torchattacks (plus)
- Robust Acc: 0.4 (175182 ms)
- AutoAttack (plus)
- Robust Acc: 0.4 (199241 ms)
- Torchattacks (rand)
- Robust Acc: 0.44 (93462 ms)
- AutoAttack (rand)
- Robust Acc: 0.44 (93450 ms)
{'fraction_non_alphanumeric': 0.1450149444918873, 'fraction_numerical': 0.055988471391972675, 'mean_word_length': 2.723569157392687, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 57, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '115968', 'license': 'MIT', 'n_tokens_mistral': 7214, 'n_tokens_neox': 6521, 'n_words': 1337, 'path': 'demos/Performance Comparison (CIFAR10).ipynb', 'repo_name': 'wanghia/adversarial-attacks-pytorch', 'size': '18736'}