<jupyter_start><jupyter_text>Generate adversarial examplesWe implement the optimization-based attacks, a naive approach by averaging the losses and an approach that extends EOT for ensemble (``Equation (7)`` in the ``ATHENA`` paper). Averaging the lossesExtend ``Equation (1)`` for ensemble by averaging the losses of individual WDs in the ensemble.\begin{equation*}max_{\delta}\frac{1}{K}\Sigma_{i=1}^{K}\left(\mathcal{L}\left(f_i, \textbf{x}+\delta, y_{true}\right)\right).\end{equation*}where $f_i$ is a weak defense in the targeted ensemble.---- How to configure the attackSame as in ``Task 1``.----* python example: `tutorials/craft_adversarial_examples.py`* main api: `attacks.attack.generate`* check tunable parameters for each attack in file `attacks/attack.py`.<jupyter_code># copied from tutorials/craft_adversarial_examples.py
def generate_ae(model, data, labels, attack_configs,
                eot=False,
                save=False, output_dir=None):
    """
    Generate adversarial examples
    :param model: WeakDefense. The targeted model.
    :param data: array. The benign samples to generate adversarial for.
    :param labels: array or list. The true labels.
    :param attack_configs: dictionary. Attacks and corresponding settings.
    :param save: boolean. True, if save the adversarial examples.
    :param output_dir: str or path. Location to save the adversarial examples.
        It cannot be None when save is True.
    :return:
    """
    img_rows, img_cols = data.shape[1], data.shape[2]
    num_attacks = attack_configs.get("num_attacks")
    data_loader = (data, labels)

    if len(labels.shape) > 1:
        labels = np.asarray([np.argmax(p) for p in labels])

    # generate attacks one by one
    for id in range(num_attacks):
        key = "configs{}".format(id)
        attack_args = attack_configs.get(key)
        
        attack_args["eot"] = eot
        data_adv = generate(model=model,
                            data_loader=data_loader,
                            attack_args=attack_args
                            )
        # predict the adversarial examples
        predictions = model.predict(data_adv)
        predictions = np.asarray([np.argmax(p) for p in predictions])

        err = error_rate(y_pred=predictions, y_true=labels)
        print(">>> error rate:", err)

        # plotting some examples
        num_plotting = min(data.shape[0], 2)
        for i in range(num_plotting):
            img = data_adv[i].reshape((img_rows, img_cols))
            plt.imshow(img, cmap='gray')
            title = '{}(EOT:{}): {}->{}'.format(attack_configs.get(key).get("description"),
                                                "ON" if eot else "OFF",
                                                labels[i],
                                                predictions[i]
                                                )
            plt.title(title)
            plt.show()
            plt.close()

        # save the adversarial example
        if save:
            if output_dir is None:
                raise ValueError("Cannot save images to a none path.")
            # save with a random name
            file = os.path.join(output_dir, "{}.npy".format(time.monotonic()))
            print("Save the adversarial examples to file [{}].".format(file))
            np.save(file, data_adv)

# loading experiment configurations
trans_configs = load_from_json(CONFIG_ROOT + "demo/athena-mnist.json")
model_configs = load_from_json(CONFIG_ROOT + "demo/model-mnist.json")
data_configs = load_from_json(CONFIG_ROOT + "demo/data-mnist.json")
attack_configs = load_from_json(CONFIG_ROOT + "demo/attack-zk-mnist.json")

# In the context of the white-box threat model,
# we use the ensemble as adversary's target model.
# load weak defenses (in this example, load a tiny pool of 3 weak defenses)
pool, _ = load_pool(trans_configs=trans_configs,
                    model_configs=model_configs,
                    active_list=True,
                    wrap=True)
# create an AVEP ensemble from the WD pool
wds = list(pool.values())
target = Ensemble(classifiers=wds, strategy=ENSEMBLE_STRATEGY.AVEP.value)

# load the benign samples
data_file = os.path.join(data_configs.get('dir'), data_configs.get('bs_file'))
data_bs = np.load(data_file)
# load the corresponding true labels
label_file = os.path.join(data_configs.get('dir'), data_configs.get('label_file'))
labels = np.load(label_file)

# generate AEs
# in this example, we generate AEs for 5 benign samples
data_bs = data_bs[:5]
labels = labels[:5]

# Normal approach
# Compute the loss w.r.t. a single input
# For an ensemble target, averaging the losses of WDs'.
# let save=True and specify an output folder to save the generated AEs
generate_ae(
    model=target, 
    data=data_bs, 
    labels=labels, 
    attack_configs=attack_configs
)<jupyter_output>>>> Loading model [../../models/cnn/model-mnist-cnn-shift_bottom_left.h5]...
WARNING:tensorflow:From /home/meng/anaconda3/envs/pytorch/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.
Instructions for updating:
Colocations handled automatically by placer.
WARNING:tensorflow:From /home/meng/anaconda3/envs/pytorch/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.
Instructions for updating:
Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.
WARNING:tensorflow:From /home/meng/anaconda3/envs/pytorch/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions [...]<jupyter_text>EOT attack* paper: [Synthesizing Robust Adversarial Examples](https://arxiv.org/pdf/1707.07397.pdf)* code GitHub repo: [EOT](https://github.com/prabhant/synthesizing-robust-adversarial-examples)* ATHENA paper: [ATHENA: A Framework based on Diverse Weak Defenses for Building Adversarial Defense](https://arxiv.org/abs/2001.00308)We extend ``EOT`` for ensemble in the ``ATHENA`` paper (``Equation (7)``). Currently, we support ``FGSM`` and ``PGD`` as the adversarial optimizer. You will need to provide the configuration for the chosen distribution.---- How to configure the distributionIn ``attack-zk-mnist.json``, add a key-value pair: key --- ``"distribution"``; value --- dictionary. configuration for the distribution.e.g.,```json"configs1": {    "attack": "pgd",    "description": "PGD_eps0.3",    "eps": 0.3,    "distribution": {      "num_samples": 500,      "transformation": "translation",      "min_offset": -0.2,      "max_offset": 0.2    }}```where ``num_samples`` is the number of random sampled transformations from the chosen distribution. For detailed explanation, please refer to the ``EOT`` and ``ATHENA`` paper.Tune the factors to generate AEs variants with various effectiveness:* parameters of the selected attack method (e.g., ``eps``, ``eps_step``, ``max_iter``, and the like, for ``PGD``).* distribution settings: ``num_samples``, ``transformation`` and corresponding settings.----* python example: `tutorials/craft_adversarial_examples.py`.* main api: `attacks.attack.generate`.* check tunable parameters for each attack in file `attacks/attack.py`.* check tunable parameters for distributions in file `attacks/distribution.py`.<jupyter_code># loading experiment configurations
trans_configs = load_from_json(CONFIG_ROOT + "demo/athena-mnist.json")
model_configs = load_from_json(CONFIG_ROOT + "demo/model-mnist.json")
data_configs = load_from_json(CONFIG_ROOT + "demo/data-mnist.json")
attack_configs = load_from_json(CONFIG_ROOT + "demo/attack-zk-mnist.json")

# load weak defenses into a pool
pool, _ = load_pool(trans_configs=trans_configs,
                    model_configs=model_configs,
                    active_list=True,
                    wrap=True)
# create an AVEP ensemble from the WD pool
wds = list(pool.values())
target = Ensemble(classifiers=wds, strategy=ENSEMBLE_STRATEGY.AVEP.value)

# load the benign samples
data_file = os.path.join(data_configs.get('dir'), data_configs.get('bs_file'))
data_bs = np.load(data_file)
# load the corresponding true labels
label_file = os.path.join(data_configs.get('dir'), data_configs.get('label_file'))
labels = np.load(label_file)

# generate AEs
# in this example, we generate AEs for 5 benign samples
data_bs = data_bs[:5]
labels = labels[:5]

# Adaptive approach (with EOT)
# Compute the loss expectation over specific distribution.
# For an ensemble target, averaging the EOT of WDs'.
# let save=True and specify an output folder to save the generated AEs
generate_ae(model=target, data=data_bs, labels=labels, eot=True, attack_configs=attack_configs)<jupyter_output>>>> Loading model [../../models/cnn/model-mnist-cnn-shift_bottom_left.h5]...
>>> Loading model [../../models/cnn/model-mnist-cnn-affine_both_stretch.h5]...
>>> Loading model [../../models/cnn/model-mnist-cnn-cartoon_mean_type3.h5]...
>>> Loaded 3 models.
>>> Generating FGSM_eps0.2(EOT:ON) examples.
>>> error rate: 0.4
{'fraction_non_alphanumeric': 0.11766612641815236, 'fraction_numerical': 0.008752025931928687, 'mean_word_length': 3.5063291139240507, 'pattern_counts': {'":': 9, '<': 7, '<?xml version=': 0, '>': 33, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '121977', 'license': 'MIT', 'n_tokens_mistral': 2904, 'n_tokens_neox': 2755, 'n_words': 781, 'path': 'notebooks/tasks/Task2_GenerateAEs_WhiteBoxThreatModel.ipynb', 'repo_name': 'Kronemeyer/project-athena', 'size': '9255'}