<jupyter_start><jupyter_text>IBALayer This notebook shows an example of the [IBALayer](https://github.com/albermax/innvestigate) for CIFAR-10.If you cannot add an additional layer to your network, have a look at [IBACopyGraph]() and [IBACopyGraphInnvestigate]().<jupyter_code># to set you cuda device
# %env CUDA_VISIBLE_DEVICES=1

%matplotlib inline

# reduce tensorflow noise
import warnings
warnings.filterwarnings("ignore")

import sys
import os
import urllib.request

from tqdm.notebook import tqdm
    
import matplotlib.pyplot as plt
import numpy as np
from PIL import Image

import tensorflow.compat.v1 as tf

import keras
import keras.backend as K

from keras.datasets import cifar10
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten
from keras.layers import Conv2D, MaxPooling2D, BatchNormalization, GlobalAveragePooling2D
import os
import tensorflow as tf
from tensorflow.python.client import device_lib


#from skimage.transform import resize

from IBA.utils import plot_saliency_map
from IBA.tensorflow_v1 import IBALayer, model_wo_softmax, to_saliency_map 

config = tf.ConfigProto()
config.gpu_options.allow_growth=True
sess = tf.Session(config=config)
keras.backend.set_session(sess)
print("TensorFlow version: {}, Keras version: {}".format(
    tf.version.VERSION, keras.__version__))
batch_size = 100
num_classes = 10
epochs = 100
data_augmentation = True

# set True to train the model yourself
run_training = True
run_training = False

model_weight_url =  "https://userpage.fu-berlin.de/leonsixt/cifar_weights.h5"

# The data, split between train and test sets:
(x_train, y_train), (x_test, y_test) = cifar10.load_data()

x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
x_train /= 255
x_test /= 255

print('x_train shape:', x_train.shape)
print(x_train.shape[0], 'train samples')
print(x_test.shape[0], 'test samples')

# Convert class vectors to binary class matrices.
y_train = keras.utils.to_categorical(y_train, num_classes)
y_test = keras.utils.to_categorical(y_test, num_classes)
# download model weights
    
! wget -O 'cifar_weights.h5' {model_weight_url} 
! echo 37d997479fbe7a0282dd0808cc1d286eb481b23279837c9ccfbec989d209eea4  cifar_weights.h5 > check_sha256sum.txt
! sha256sum --check check_sha256sum.txt
K.clear_session()
model = Sequential()
model.add(Conv2D(32, (3, 3), padding='same', name='conv1', use_bias=False,
                 input_shape=x_train.shape[1:]))
model.add(BatchNormalization(name='bn1'))
model.add(Activation('relu', name='relu1'))

model.add(Conv2D(64, (5, 5), padding='same', name='conv2', use_bias=False))
model.add(BatchNormalization(name='bn2'))
model.add(Activation('relu', name='relu2'))
model.add(MaxPooling2D(pool_size=(2, 2), name='pool2'))    #  8

# just add IBALayer where you want to explain the model.
# you could even add multiple IBALayers to a single model.
model.add(IBALayer())
model.add(Conv2D(128, (5, 5), padding='same', name='conv3', use_bias=False))
model.add(BatchNormalization(name='bn3'))
model.add(Activation('relu', name='relu3'))

model.add(Conv2D(128, (5, 5), padding='same', name='conv4', use_bias=False))
model.add(Activation('relu', name='relu4'))
model.add(GlobalAveragePooling2D(name='pool4'))   

model.add(Dropout(0.5, name='dropout1'))
model.add(Dense(1024, name='fc1'))
model.add(Activation('relu', name='relu5'))

model.add(Dropout(0.5, name='dropout2'))
model.add(Dense(num_classes, name='fc2'))
model.add(Activation('softmax', name='softmax'))
if not run_training:
    print("loading weights")
    model.load_weights('cifar_weights.h5')
opt = keras.optimizers.Adam()

model.compile(loss='categorical_crossentropy',
              optimizer=opt,
              metrics=['accuracy'])

# Score trained model.
scores = model.evaluate(x_test, y_test, verbose=1)
print('Test loss:', scores[0])
print('Test accuracy:', scores[1])
if run_training: 
    datagen = ImageDataGenerator(
        rotation_range=20,  
        width_shift_range=0.1,
        height_shift_range=0.1,
        horizontal_flip=True,  # randomly flip images
    )

    datagen.fit(x_train)

    # Fit the model on the batches generated by datagen.flow().
    hist = model.fit_generator(
        datagen.flow(x_train, y_train, batch_size=batch_size), 
        epochs=epochs, 
        steps_per_epoch=len(x_train) // batch_size,
        validation_data=(x_test, y_test), workers=4)
    
    model.save_weights("cifar_weights.h5")
# Score trained model.
scores = model.evaluate(x_test, y_test, verbose=1)
print('Test loss:', scores[0])
print('Test accuracy:', scores[1])
model_logits = model_wo_softmax(model)
iba = model.layers[7]
target = iba.set_classification_loss(model_logits.output)
# ensure model is in eval mode
K.set_learning_phase(0)

# estimate mean, std on 5000 samples
for img in tqdm(x_train[:5000]):
    iba.fit({model.input: img[None]})
rows = 2
cols = 6
fig, ax = plt.subplots(rows, cols, figsize=(2.5*cols, 2.*rows))

for i, ax0, ax1 in zip(range(rows*cols//2), ax.flatten()[::2], ax.flatten()[1::2]):
    img = x_test[i:i+1]
    ax0.imshow(img[0])
    ax0.set_xticks([])
    ax0.set_yticks([])
    target = y_test[i].nonzero()[0]
    capacity = iba.analyze({model.input: img, iba.target: target})
    saliency_map = to_saliency_map(capacity, shape=(32, 32))

    plot_saliency_map(saliency_map, ax=ax1, colorbar_size=0.15, colorbar_fontsize=10)<jupyter_output><empty_output><jupyter_text>Access to internal values You can access all intermediate values of the optimzation through the `iba.get_report()` method.To store the intermediate values, you have to call either `iba.collect_all()` or `iba.collect(*var_names)` before running `iba.analyze(..)`.<jupyter_code># collect all intermediate tensors
iba.collect_all()

# storing all tensors can slow down the optimization. 
# you can also select to store only specific ones:
# iba.collect("alpha", "model_loss")
# to only collect a subset all all tensors

# run analyze

i = 4
img = x_test[i][None]
target = y_test[i].nonzero()[0]
capacity = iba.analyze({model.input: img, iba.target: target})
saliency_map = to_saliency_map(capacity, shape=(32, 32))

# get all saved variables
report = iba.get_report()<jupyter_output><empty_output><jupyter_text>`report` is an `OrderedDict`  which maps each `iteration` to a dictionray of `{var_name, var_value}`.The `init` iteration is computed without an optimizer update. Values not changing such as the feature values are only included in the `init` iteration.The `final` iteration is again computed without an optimizer update.<jupyter_code>print("iterations:", list(report.keys()))<jupyter_output><empty_output><jupyter_text>Print all available tensors in the `init` iteration:<jupyter_code>print("{:<30} {:}".format("name:", "shape"))
print()
for name, val in report['init'].items():
    print("{:<30} {:}".format(name + ":", str(val.shape)))<jupyter_output><empty_output><jupyter_text>Losses during optimization<jupyter_code>fig, ax = plt.subplots(1, 2, figsize=(8, 3))
ax[0].set_title("cross entrop loss")
ax[0].plot(list(report.keys()), [it['model_loss'] for it in report.values()])

ax[1].set_title("mean capacity")
ax[1].plot(list(report.keys()), [it['capacity_mean'] for it in report.values()])<jupyter_output><empty_output><jupyter_text>Distribution of alpha (pre-softmax) values per iteraton<jupyter_code>cols = 6
rows = len(report) // cols

fig, axes = plt.subplots(rows, cols, figsize=(2.8*cols, 2.2*rows))

for ax, (it, values) in zip(axes.flatten(), report.items()):
    ax.hist(values['alpha'].flatten(), log=True, bins=20)
    ax.set_title("iteration: " + str(it))
    
plt.subplots_adjust(wspace=0.3, hspace=0.5)

fig.suptitle("distribution of alpha (pre-softmax) values per iteraton.", y=1)
plt.show()<jupyter_output><empty_output><jupyter_text>Distributiuon of the final capacity<jupyter_code>plt.hist(report['final']['capacity'].flatten(), bins=20, log=True)
plt.title("Distributiuon of the final capacity")
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1378582950738305, 'fraction_numerical': 0.02754684203995533, 'mean_word_length': 5.824724809483489, 'pattern_counts': {'":': 1, '<': 31, '<?xml version=': 0, '>': 30, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '185026', 'license': 'MIT', 'n_tokens_mistral': 2985, 'n_tokens_neox': 2828, 'n_words': 651, 'path': 'notebooks/tensorflow_IBALayer_cifar.ipynb', 'repo_name': 'karl-schulz/IBA', 'size': '8059'}