<jupyter_start><jupyter_text>Checking the ANN<jupyter_code>import os
cwd = os.getcwd()
cwd
from keras.models import load_model
import dnns
import numpy as np
from dnns.noisy_sgd import NoisySGD

# load model
model = load_model('../trained_model_of_lenet_300_100_relu_crossent_noisy_sgd_retest_final_dense.h5', custom_objects={'NoisySGD':NoisySGD})
# summarize model.
model.summary()
# load dataset
from dnns.load_dataset import load_and_preprocess_dataset
is_output_categorical = True
dataset_info = load_and_preprocess_dataset(
    'mnist', categorical_output=is_output_categorical)
x_train, y_train = dataset_info['train']
x_test, y_test = dataset_info['test']
img_rows, img_cols = dataset_info['img_dims']
input_shape = dataset_info['input_shape']
num_classes = dataset_info['num_classes']

# reshape input to flatten data
x_train = x_train.reshape(x_train.shape[0], 1, np.prod(x_train.shape[1:]))
np.savez('x_train', x_train)
x_test = x_test.reshape(x_test.shape[0], 1, np.prod(x_test.shape[1:]))
np.savez('x_test', x_test)
x_norm = x_train[:10000].reshape(10000, 1, np.prod(x_train.shape[1:]))
np.savez('x_norm', x_norm)

np.savez('y_train', y_train)
np.savez('y_test', y_test)

score = model.evaluate(x_test, y_test, verbose=1, batch_size=10)
print('Test Loss:', score[0])
print('Test Accuracy:', score[1])<jupyter_output><empty_output><jupyter_text>Running the Toolbox<jupyter_code>from snntoolbox.bin.run import run_toolbox
t_stim = 1000
testing_examples = 10
config = """[paths]
dataset_path = %(path_wd)s
filename_ann = trained_model_of_lenet_300_100_relu_crossent_noisy_sgd_retest_final_dense
runlabel = lenet_dense_dt_0.1_normalised_99

[tools]
evaluate_ann = False
normalize = False
only_serialise = True

[normalization]
percentile = 99

[simulation]
simulator = spiNNaker
duration = 1000
num_to_test = 5
batch_size = 5
keras_backend = tensorflow
dt = 0.1

[cell]
tau_syn_E = 0.2
tau_syn_I = 0.2
v_thresh = 1

[input]
poisson_input = True


[output]
log_vars = {'all'}
plot_vars = {'all'}"""

file='config'
with open(file, 'w') as filetowrite:
    filetowrite.write(config)

f = open("config","r")
#necessary to fully reset simulator
from spinn_front_end_common.utilities import globals_variables
globals_variables.unset_simulator()

run_toolbox(config_filepath='config',terminal=True)
#complains because it tries to do data analysis
#in future it would be better to put serialisation inside the toolbox<jupyter_output><empty_output><jupyter_text>Running the serialised model<jupyter_code>#necessary to fully reset simulator
from spinn_front_end_common.utilities import globals_variables
globals_variables.unset_simulator()

extra_args = ['lenet_dense_dt_1_not_normalised_serialised', '--t_stim', str(t_stim), '--testing_examples',\
              str(testing_examples), '--result_filename', 'output_data', '--result_dir', '.']
import pynn_object_serialisation.experiments.mnist_testing.mnist_testing as mnist_testing
from pynn_object_serialisation.experiments.mnist_testing.mnist_argparser import parser

new_args = parser.parse_args(extra_args) 
mnist_testing.run(new_args)<jupyter_output><empty_output><jupyter_text>Looking at the output<jupyter_code>from pynn_object_serialisation.OutputDataProcessor import OutputDataProcessor
np.set_printoptions(suppress=True)
output = OutputDataProcessor('/mnt/snntoolbox/snn_toolbox_private/examples/models/test_examples/lenet_dense/results/output_data_900.npz')
spikes = spikes['03Dense_10']
spikes = spikes + [0,1]
spikes + [0,1]
output.t_stim
output.y_test.shape
output.accuracy
output.y_pred == output.y_test
output.plot_bin(0, output.layer_names[0], shape = (28,28))
output.layer_names
output.get_bounds(1)<jupyter_output><empty_output><jupyter_text>Multiple Evaluations This is used to make graphs<jupyter_code>#Being run separately
import multiprocessing
import sys
import os
from multiprocessing import Pool
from time import time
import numpy as np

def EvaluateModel(t_stim, testing_examples):
    current = multiprocessing.current_process()
    print('Started {}'.format(current))
    f_name = "errorlog/" + current.name +"_stdout.txt"
    g_name = "errorlog/" + current.name + "_stderror.txt"
    f = open(f_name, 'w')
    g = open(g_name, 'w')
    old_stdout = sys.stdout
    old_stderr = sys.stderr
    sys.stdout = f
    sys.stderr = g
        
    from spinn_front_end_common.utilities import globals_variables
    globals_variables.unset_simulator()

    extra_args = ['lenet_dense_dt_0.1_not_normalised_serialised', '--t_stim', str(t_stim), '--testing_examples',\
                  str(testing_examples), '--result_filename', 'output_data_'+str(t_stim), '--result_dir', 'results',\
                  '--chunk_size', '20']
    import pynn_object_serialisation.experiments.mnist_testing.mnist_testing as mnist_testing
    from pynn_object_serialisation.experiments.mnist_testing.mnist_argparser import parser

    new_args = parser.parse_args(extra_args) 
    mnist_testing.run(new_args)
    sys.stdout = old_stdout
    sys.stderr = old_stderr
    print("Run for {} completed".format(t_stim))
    
    return
po = Pool(15)
range_input = np.array(range(1000,1600,100))
input_data = [(i,100) for i in range_input]
#output = po.starmap(EvaluateModel, input_data)
print('Done!')
test = OutputDataProcessor('/mnt/snntoolbox/snn_toolbox_private/examples/models/test_examples/lenet_dense/results/output_data_1000.npz')
test.y_test
test.y_pred
import glob, os
from pynn_object_serialisation.OutputDataProcessor import OutputDataProcessor
dt_01_accuracies = []

os.chdir("/mnt/snntoolbox/snn_toolbox_private/examples/models/test_examples/lenet_dense/results/")
for file in glob.glob("output_data_[0-9]*.npz"):
    output = OutputDataProcessor('/mnt/snntoolbox/snn_toolbox_private/examples/models/test_examples/lenet_dense/results/'+file)
    dt_01_accuracies.append([output.t_stim,output.accuracy])
    print(file, output.t_stim,output.accuracy)
dt_01_accuracies = np.array(dt_01_accuracies).astype(float)
import glob, os
from pynn_object_serialisation.OutputDataProcessor import OutputDataProcessor
dt_1_accuracies = []

os.chdir("/mnt/snntoolbox/snn_toolbox_private/examples/models/test_examples/lenet_dense/results/")
for file in glob.glob("output_data_dt_1_[0-9]*.npz"):
    output = OutputDataProcessor('/mnt/snntoolbox/snn_toolbox_private/examples/models/test_examples/lenet_dense/results/'+file)
    dt_1_accuracies.append([output.t_stim,output.accuracy])
    print(file, output.t_stim,output.accuracy)
dt_1_accuracies = np.array(dt_1_accuracies).astype(float)
dt_01_accuracies
import matplotlib.pyplot as plt
import numpy as np

accuracies = np.array(accuracies)
#dt_1_accuracies = dt_1_accuracies[:-1,:]


fig = plt.figure(dpi=300)
ax = plt.subplot(111)
ax.scatter(dt_01_accuracies[:,0], 100*dt_01_accuracies[:,1], label='SNN 0.1ms timestep')
ax.scatter(dt_1_accuracies[:,0], 100*dt_1_accuracies[:,1], label='SNN 1ms timestep')
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
ax.set_xlabel('Stimulation time (ms)')
ax.set_ylabel('Accuracy (%)')
ax.axhline(y=97.9, color='r', linestyle='-', label='ANN (97.9%)')
ax.set_ylim([0,100])
ax.set_xlim([0,10500])
ax.legend(loc='center left', bbox_to_anchor=(1, 0.5))
range_input = np.array([50,60,70,80,90,95,99])
#range_input = np.array([90,95])

from pynn_object_serialisation.OutputDataProcessor import OutputDataProcessor
dt_01_norm_accuracies = []
for norm in range_input:
    accuracy = OutputDataProcessor('/mnt/snntoolbox/snn_toolbox_private/examples/models/test_examples/lenet_dense/results/output_data_dt_01_normalised_'+str(norm)+'.npz').accuracy
    dt_01_norm_accuracies.append([norm,accuracy])

dt_01_norm_accuracies = np.array(dt_01_norm_accuracies)


range_input = np.array([50,60,70,80,90,95,99])
from pynn_object_serialisation.OutputDataProcessor import OutputDataProcessor
dt_1_norm_accuracies = []
for norm in range_input:
    accuracy = OutputDataProcessor('/mnt/snntoolbox/snn_toolbox_private/examples/models/test_examples/lenet_dense/results/output_data_dt_1_normalised_'+str(norm)+'.npz').accuracy
    dt_1_norm_accuracies.append([norm,accuracy])

dt_1_norm_accuracies = np.array(dt_1_norm_accuracies)
out = OutputDataProcessor('/mnt/snntoolbox/snn_toolbox_private/examples/models/test_examples/lenet_dense/results/output_data_dt_01_normalised_50.npz')
out.y_pred
out.y_test
import matplotlib.pyplot as plt
import numpy as np

fig = plt.figure(dpi=300)
ax = plt.subplot(111)
ax.scatter(dt_1_norm_accuracies[:,0], 100*dt_1_norm_accuracies[:,1], label='SNN 1ms timestep')
#ax.axhline(y=dt_1_accuracies[])
ax.scatter(dt_01_norm_accuracies[:,0], 100*dt_01_norm_accuracies[:,1], label='SNN 0.1ms timestep')

box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
ax.set_xlabel('Normalisation percentile')
ax.set_ylabel('Accuracy (%)')
ax.axhline(y=97.9, color='r', linestyle='-', label='ANN (97.9%)')
ax.set_ylim([0,100])
ax.set_xlim([40, 100])
ax.legend(loc='center left', bbox_to_anchor=(1, 0.5))
dt_01_norm_accuracies[:,0]
dt_1_accuracies[:,0]
from pynn_object_serialisation.OutputDataProcessor import OutputDataProcessor
sparse_test = OutputDataProcessor('/mnt/snntoolbox/snn_toolbox_private/examples/models/test_examples/lenet_sparse/results/test_1000ms_10_examples_unnormalised_2.npz')
sparse_test.accuracy
sparse_test.y_pred
sparse_test.y_test
sparse_test_long = OutputDataProcessor('/mnt/snntoolbox/snn_toolbox_private/examples/models/test_examples/lenet_sparse/results/test_10000ms_10_examples.npz')
sparse_test_long.accuracy
sparse_test.y_pred
sparse_test.dt
sparse_test_long.get_counts(5, sparse_test.layer_names[3], 10)
sparse_test_long.plot_bin(5, sparse_test_long.layer_names[-1], shape=(1,10))
sparse_test.y_test<jupyter_output><empty_output><jupyter_text>Input ANN check activation values<jupyter_code>from keras.models import load_model
import dnns
import numpy as np
from dnns.noisy_sgd import NoisySGD
from dnns.sparse_layer import Sparse
custom_objects = {'NoisySGD':NoisySGD, 'Sparse':Sparse}
# load model
dense_model = load_model('/mnt/snntoolbox/snn_toolbox_private/examples/models/test_examples/lenet_dense/trained_model_of_lenet_300_100_relu_crossent_noisy_sgd_retest_final_dense.h5', custom_objects=custom_objects)
sparse_model = load_model('/mnt/snntoolbox/snn_toolbox_private/examples/models/test_examples/lenet_sparse/trained_model_of_lenet_300_100_relu_crossent_noisy_sgd_retest.h5', custom_objects=custom_objects)
dense_model.summary()
sparse_model.summary()
print(sparse_model.layers[-1].activation)
plt.hist(dense_model.layers[3].get_weights()[0].flatten())
some_weights = sparse_model.layers[-1].get_weights()[2] * sparse_model.layers[-1].get_weights()[0]
plt.hist(some_weights[(abs(some_weights) > 0)].flatten(), bins=100)
np.max(some_weights[some_weights<0])
from keras.datasets import mnist
import matplotlib.pyplot as plt

(x_train, y_train), (x_test, y_test) = mnist.load_data()
from keras.models import Model
from keras.activations import relu, linear, tanh, softsign, exponential,sigmoid

from vis.utils.utils import apply_modifications

model = sparse_model
print([layer.name for layer in model.layers])

def update_layer_activation(model, activation, index=-1, custom_objects=None):
    model.layers[index].activation = activation
    return apply_modifications(model, custom_objects=custom_objects)


layer_name = 'sparse_3'
intermediate_layer_model = Model(inputs=model.input,
                                 outputs=model.get_layer(layer_name).output)

intermediate_layer_model = update_layer_activation(intermediate_layer_model, linear, custom_objects=custom_objects)


print(intermediate_layer_model.layers[-1].activation)
intermediate_layer_model.summary()
intermediate_output = intermediate_layer_model.predict(x_test[:10].reshape((10,1,784)))
model_output = model.predict(x_test[:10].reshape((10,1,784)))
intermediate_output[3]
intermediate_output = intermediate_layer_model.predict(x_test[:10].reshape((10,1,784)))
plt.imshow(intermediate_output[9].reshape(10,-1))
plt.colorbar()
plt.imshow(x_test[0].reshape((28,28)))
x_test[0].flatten().shape
plt.imshow(sparse_test.get_counts(9, sparse_test.layer_names[2], 100).reshape((10,-1)))
plt.colorbar()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12140287769784172, 'fraction_numerical': 0.030493786788750817, 'mean_word_length': 7.3787671232876715, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '83774', 'license': 'MIT', 'n_tokens_mistral': 4755, 'n_tokens_neox': 4459, 'n_words': 666, 'path': 'examples/models/test_examples/lenet_dense/MNIST workflow.ipynb', 'repo_name': 'ej159/snn_toolbox', 'size': '12232'}