<jupyter_start><jupyter_text>MNIST - sensitivity to input orientation<jupyter_code>mnist = tf.keras.datasets.mnist

(X_train, y_train), (X_test, y_test) = mnist.load_data()
X_train, X_test = X_train / 255.0, X_test / 255.0
X_train = X_train[:, :, :, np.newaxis]
X_test = X_test[:, :, :, np.newaxis]


def make_model_mnist(): 
    input_shape = (28, 28, 1)
    model = keras.models.Sequential()
    model.add(layers.Conv2D(32, kernel_size=(3, 3),
                     activation='relu',
                     input_shape=input_shape))
    model.add(layers.Conv2D(64, (3, 3), activation='relu'))
    model.add(layers.MaxPooling2D(pool_size=(2, 2)))
    model.add(layers.Dropout(0.25))
    model.add(layers.Flatten())
    model.add(layers.Dense(128, activation='relu'))
    model.add(layers.Dropout(0.5))
    model.add(layers.Dense(10, activation='softmax'))

    model.compile(optimizer='adam',
                      loss='sparse_categorical_crossentropy',
                      metrics=['accuracy'])
    return model
model = make_model_mnist()
batch_size = 128
num_epoch = 10
savemodel = False # Handy to repeat the experiment
if savemodel and os.path.exists("/tmp/mnist.h5"):
    model.load_weights("/tmp/mnist.h5")
else:
    model_log = model.fit(X_train, y_train,
              batch_size=batch_size,
              epochs=num_epoch,
              verbose=1,
              validation_data=(X_test, y_test))

    if savemodel:
        model.save("/tmp/mnist.h5")
# Should be 99% accurate
model.evaluate(X_test, y_test, verbose=2)
# Define a function to rotate a digit example

def rotate(X, angle):
    rotated = scipy.ndimage.rotate(X, angle, axes=(1,2))
    w = int(rotated.shape[1]/2)
    return rotated[:, w-14:w+14, w-14:w+14, :]<jupyter_output><empty_output><jupyter_text>Now, create a probe object to wrap the model, then pass it the rotate function to test the sensitivity to rotation using the test set; we'll test rotation from 0 to 180 degrees.<jupyter_code>mnist_probe = sensie.Probe(model=model)
pfunc = rotate
rot_test = mnist_probe.predict_and_measure_perturbed(X_test, y_test, pfunc, p_min=0, p_max=180, steps=20, 
                              label='rotation', plot=True, ci=False)<jupyter_output>[########################################] 100% (20/20)    180.00<jupyter_text>Yes, this looks like a significant effect. This could be viewed as a weakness of the model. Let's retrain with rotatedexamples and see how things change.<jupyter_code>X_train_rot = np.empty_like(X_train)
X_test_rot = np.empty_like(X_test)
for i in range(X_train.shape[0]):
    angle = np.random.random()*180
    X_train_rot[i:i+1] = rotate(X_train[i:i+1], angle)
    if i < X_test.shape[0]:
        X_test_rot[i:i+1] = rotate(X_test[i:i+1], angle)

model_log = model.fit(X_train_rot, y_train,
          batch_size=batch_size,
          epochs=num_epoch,
          verbose=1,
          validation_data=(X_test_rot, y_test))
rot_test = mnist_probe.predict_and_measure_perturbed(X_test, y_test, pfunc, p_min=0, p_max=180, steps=10, 
                              label='rotation', plot=True)<jupyter_output>[########################################] 100% (10/10)    180.00
{'fraction_non_alphanumeric': 0.1527514231499051, 'fraction_numerical': 0.03478810879190386, 'mean_word_length': 3.5973837209302326, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '592508', 'license': 'MIT', 'n_tokens_mistral': 1199, 'n_tokens_neox': 1114, 'n_words': 224, 'path': 'examples/MNIST example.ipynb', 'repo_name': 'coljac/sensie', 'size': '3162'}