<jupyter_start><jupyter_text>This notebook plots a comparison of pre- and post-activations in CNN-deep before and after training (Supplementary Fig. 10)<jupyter_code>import os
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
from tensorflow import keras
import helper
# load data
data_path = '../data/synthetic_dataset.h5'
data = helper.load_dataset(data_path)
x_train, y_train, x_valid, y_valid, x_test, y_test = data

# load ground truth values
test_model = helper.load_synthetic_models(data_path, dataset='test')
X = x_test
X_model = test_model
print(X.shape)<jupyter_output>(6000, 200, 4)<jupyter_text>Plot pre and post activation values for 1st convolutional layer prior to training<jupyter_code>num_trials = 1
model_names = ['cnn-deep']
activations = ['exponential', 'relu']

results_path = os.path.join('../results', 'task1')
params_path = os.path.join(results_path, 'model_params')

results = {}
results2 = {}
for model_name in model_names:
    for activation in activations:
        results[model_name+'_'+activation] = []
        results2[model_name+'_'+activation] = []
        for trial in range(num_trials):
            keras.backend.clear_session()
            
            # load model (it is randomly initialized)
            model, name = helper.load_model(model_name, 
                                            activation=activation, 
                                            dropout=True, 
                                            l2=1e-6, 
                                            bn=True,
                                            input_shape=200)
            base_name = name+'_'+activation
            name = base_name+'_'+str(trial)
            print('model: ' + name)

            # after activations 
            intermediate = keras.Model(inputs=model.inputs, outputs=model.layers[3].output)
            results[model_name+'_'+activation].append(intermediate.predict(X))
        
            # before activations         
            intermediate2 = keras.Model(inputs=model.inputs, outputs=model.layers[2].output)
            results2[model_name+'_'+activation].append(intermediate2.predict(X))<jupyter_output><empty_output><jupyter_text>Only get activations for the first 100 sequences<jupyter_code>x = np.reshape(np.array(results['cnn-deep_exponential'])[0][:100], [-1])
y = np.reshape(np.array(results['cnn-deep_relu'])[0][:100], [-1])
x2 = np.reshape(np.array(results2['cnn-deep_exponential'])[0][:100], [-1])
y2 = np.reshape(np.array(results2['cnn-deep_relu'])[0][:100], [-1])<jupyter_output><empty_output><jupyter_text>Plot the activations of CNN-deep with exponential activations prior to training<jupyter_code># definitions for the axes
left, width = 0.1, 0.65
bottom, height = 0.1, 0.65
spacing = 0.005

rect_scatter = [left, bottom, width, height]
rect_histx = [left, bottom + height + spacing, width, 0.2]
rect_histy = [left + width + spacing, bottom, 0.2, height]

# start with a rectangular Figure
fig = plt.figure(figsize=(8, 8))

ax_scatter = plt.axes(rect_scatter)
ax_scatter.tick_params(direction='in', top=True, right=True)
plt.xlabel('Pre-activation', fontsize=14)
plt.ylabel('Post-activation', fontsize=14)

ax_histx = plt.axes(rect_histx)
ax_histx.tick_params(direction='in', labelbottom=False)
ax_histy = plt.axes(rect_histy)
ax_histy.tick_params(direction='in', labelleft=False)
ax_scatter.set_yticks([0,100,200,300,400])
ax_scatter.set_yticklabels([0,100,200,300,400], fontsize=14)
ax_scatter.set_xticks([-6, -4, -2, 0, 2, 4, 6])
ax_scatter.set_xticklabels([-6, -4, -2, 0, 2, 4, 6], fontsize=14)

# the scatter plot:
ax_scatter.scatter(x2, x, alpha=0.1, rasterized=True)

# now determine nice limits by hand:
binwidth = 0.25
ax_scatter.set_xlim((-6, 7))
ax_scatter.set_ylim((-0, 450))

bins = np.arange(-6,6 + binwidth, binwidth)
ax_histx.hist(x2, bins=bins)
ax_histx.spines['top'].set_visible(False)
ax_histx.spines['right'].set_visible(False)
ax_histx.set_yticks([40000,80000])
ax_histx.set_yticklabels([40000,80000], fontsize=14)

ax_histy.hist(x, bins=30, orientation='horizontal')
ax_histy.spines['top'].set_visible(False)
ax_histy.spines['right'].set_visible(False)
ax_histy.set_xticks([150000])
ax_histy.set_xticklabels([150000], fontsize=14)

ax_histx.set_xlim(ax_scatter.get_xlim())
ax_histy.set_ylim(ax_scatter.get_ylim())

outfile = os.path.join(results_path, 'pre-post_activation_exp_random.pdf')
fig.savefig(outfile, format='pdf', dpi=200, bbox_inches='tight')<jupyter_output>model: cnn-deep_do_l2_bn_exponential_0
model: cnn-deep_do_l2_bn_exponential_1
model: cnn-deep_do_l2_bn_exponential_2
model: cnn-deep_do_l2_bn_exponential_3
model: cnn-deep_do_l2_bn_exponential_4
model: cnn-deep_do_l2_bn_exponential_5
model: cnn-deep_do_l2_bn_exponential_6
model: cnn-deep_do_l2_bn_exponential_7
model: cnn-deep_do_l2_bn_exponential_8
model: cnn-deep_do_l2_bn_exponential_9
model: cnn-deep_do_l2_bn_relu_0
model: cnn-deep_do_l2_bn_relu_1
model: cnn-deep_do_l2_bn_relu_2
model: cnn-deep_do_l2_bn_relu_3
model: cnn-deep_do_l2_bn_relu_4
model: cnn-deep_do_l2_bn_relu_5
model: cnn-deep_do_l2_bn_relu_6
model: cnn-deep_do_l2_bn_relu_7
model: cnn-deep_do_l2_bn_relu_8
model: cnn-deep_do_l2_bn_relu_9<jupyter_text>Plot the activations of CNN-deep with relu activations prior to training<jupyter_code># definitions for the axes
left, width = 0.1, 0.65
bottom, height = 0.1, 0.65
spacing = 0.005

rect_scatter = [left, bottom, width, height]
rect_histx = [left, bottom + height + spacing, width, 0.2]
rect_histy = [left + width + spacing, bottom, 0.2, height]

# start with a rectangular Figure
fig = plt.figure(figsize=(8, 8))

ax_scatter = plt.axes(rect_scatter)
ax_scatter.tick_params(direction='in', top=True, right=True)
plt.xlabel('Pre-activation', fontsize=14)
plt.ylabel('Post-activation', fontsize=14)

ax_histx = plt.axes(rect_histx)
ax_histx.tick_params(direction='in', labelbottom=False)
ax_histy = plt.axes(rect_histy)
ax_histy.tick_params(direction='in', labelleft=False)
ax_scatter.set_yticks([0,1,2])
ax_scatter.set_yticklabels([0,1,2], fontsize=14)
ax_scatter.set_xticks([-6, -4, -2, 0, 2, 4, 6])
ax_scatter.set_xticklabels([-6, -4, -2, 0, 2, 4, 6], fontsize=14)

# the scatter plot:
ax_scatter.scatter(y2,y, alpha=0.1, rasterized=True)

# now determine nice limits by hand:
binwidth = 0.25
ax_scatter.set_xlim((-6, 7))
ax_scatter.set_ylim((-0, 3))

bins = np.arange(-6,6 + binwidth, binwidth)
ax_histx.hist(y2, bins=bins)
ax_histx.spines['top'].set_visible(False)
ax_histx.spines['right'].set_visible(False)
ax_histx.set_yticks([40000, 80000])
ax_histx.set_yticklabels([40000, 80000], fontsize=14)

ax_histy.hist(y, bins=20, orientation='horizontal')
ax_histy.spines['top'].set_visible(False)
ax_histy.spines['right'].set_visible(False)
ax_histy.set_xticks([300000])
ax_histy.set_xticklabels([300000], fontsize=14)

ax_histx.set_xlim(ax_scatter.get_xlim())
ax_histy.set_ylim(ax_scatter.get_ylim())

outfile = os.path.join(results_path, 'pre-post_activation_relu_random.pdf')
fig.savefig(outfile, format='pdf', dpi=200, bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>Now plot pre and post activations in first layer after training<jupyter_code>num_trials = 1
model_names = ['cnn-deep']# deep', 'cnn-25', 'cnn-4'] #
activations = ['exponential', 'relu']

results_path = utils.make_directory('../results', 'synthetic')
params_path = utils.make_directory(results_path, 'model_params')

results = {}
results2 = {}
for model_name in model_names:
    for activation in activations:
        results[model_name+'_'+activation] = []
        results2[model_name+'_'+activation] = []
        for trial in range(num_trials):
            keras.backend.clear_session()
            
            # load model
            model, name = helper.load_model(model_name, 
                                            activation=activation, 
                                            dropout=True, 
                                            l2=1e-6, 
                                            bn=True,
                                            input_shape=200)
            base_name = name+'_'+activation
            name = base_name+'_'+str(trial)
            print('model: ' + name)

            # load pretrained weights
            weights_path = os.path.join(params_path, name+'.hdf5')
            model.load_weights(weights_path)            
            
            # after activations 
            intermediate = keras.Model(inputs=model.inputs, outputs=model.layers[3].output)
            results[model_name+'_'+activation].append(intermediate.predict(X))
        
            # before activations         
            intermediate2 = keras.Model(inputs=model.inputs, outputs=model.layers[2].output)
            results2[model_name+'_'+activation].append(intermediate2.predict(X))
        
x = np.reshape(np.array(results['cnn-deep_exponential'])[0][:100], [-1])
y = np.reshape(np.array(results['cnn-deep_relu'])[0][:100], [-1])
x2 = np.reshape(np.array(results2['cnn-deep_exponential'])[0][:100], [-1])
y2 = np.reshape(np.array(results2['cnn-deep_relu'])[0][:100], [-1])<jupyter_output><empty_output><jupyter_text>Plot the activations of CNN-deep with exponential activations after training<jupyter_code># definitions for the axes
left, width = 0.1, 0.65
bottom, height = 0.1, 0.65
spacing = 0.005

rect_scatter = [left, bottom, width, height]
rect_histx = [left, bottom + height + spacing, width, 0.2]
rect_histy = [left + width + spacing, bottom, 0.2, height]

# start with a rectangular Figure
fig = plt.figure(figsize=(8, 8))

ax_scatter = plt.axes(rect_scatter)
ax_scatter.tick_params(direction='in', top=True, right=True)
plt.xlabel('Pre-activation', fontsize=14)
plt.ylabel('Post-activation', fontsize=14)

ax_histx = plt.axes(rect_histx)
ax_histx.tick_params(direction='in', labelbottom=False)
ax_histy = plt.axes(rect_histy)
ax_histy.tick_params(direction='in', labelleft=False)
ax_scatter.set_yticks([0,100,200,300,400])
ax_scatter.set_yticklabels([0,100,200,300,400], fontsize=14)
ax_scatter.set_xticks([-6, -4, -2, 0, 2, 4, 6])
ax_scatter.set_xticklabels([-6, -4, -2, 0, 2, 4, 6], fontsize=14)

# the scatter plot:
ax_scatter.scatter(x2, x, alpha=0.1, rasterized=True)

# now determine nice limits by hand:
binwidth = 0.25
ax_scatter.set_xlim((-6, 7))
ax_scatter.set_ylim((-0, 450))

bins = np.arange(-6,6 + binwidth, binwidth)
ax_histx.hist(x2, bins=bins)
ax_histx.spines['top'].set_visible(False)
ax_histx.spines['right'].set_visible(False)
ax_histx.set_yticks([40000,80000])
ax_histx.set_yticklabels([40000,80000], fontsize=14)

ax_histy.hist(x, bins=30, orientation='horizontal')
ax_histy.spines['top'].set_visible(False)
ax_histy.spines['right'].set_visible(False)
ax_histy.set_xticks([150000])
ax_histy.set_xticklabels([150000], fontsize=14)

ax_histx.set_xlim(ax_scatter.get_xlim())
ax_histy.set_ylim(ax_scatter.get_ylim())

outfile = os.path.join(results_path, 'pre-post_activation_exp_trained.pdf')
fig.savefig(outfile, format='pdf', dpi=200, bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>Plot the activations of CNN-deep with relu activations after training<jupyter_code># definitions for the axes
left, width = 0.1, 0.65
bottom, height = 0.1, 0.65
spacing = 0.005

rect_scatter = [left, bottom, width, height]
rect_histx = [left, bottom + height + spacing, width, 0.2]
rect_histy = [left + width + spacing, bottom, 0.2, height]

# start with a rectangular Figure
fig = plt.figure(figsize=(8, 8))

ax_scatter = plt.axes(rect_scatter)
ax_scatter.tick_params(direction='in', top=True, right=True)
plt.xlabel('Pre-activation', fontsize=14)
plt.ylabel('Post-activation', fontsize=14)

ax_histx = plt.axes(rect_histx)
ax_histx.tick_params(direction='in', labelbottom=False)
ax_histy = plt.axes(rect_histy)
ax_histy.tick_params(direction='in', labelleft=False)
ax_scatter.set_yticks([0,1,2])
ax_scatter.set_yticklabels([0,1,2], fontsize=14)
ax_scatter.set_xticks([-6, -4, -2, 0, 2, 4, 6])
ax_scatter.set_xticklabels([-6, -4, -2, 0, 2, 4, 6], fontsize=14)

# the scatter plot:
ax_scatter.scatter(y2,y, alpha=0.1, rasterized=True)

# now determine nice limits by hand:
binwidth = 0.25
ax_scatter.set_xlim((-6, 7))
ax_scatter.set_ylim((-0, 3))

bins = np.arange(-6,6 + binwidth, binwidth)
ax_histx.hist(y2, bins=bins)
ax_histx.spines['top'].set_visible(False)
ax_histx.spines['right'].set_visible(False)
ax_histx.set_yticks([20000,40000, 60000])
ax_histx.set_yticklabels([20000,40000, 60000], fontsize=14)

ax_histy.hist(y, bins=20, orientation='horizontal')
ax_histy.spines['top'].set_visible(False)
ax_histy.spines['right'].set_visible(False)
ax_histy.set_xticks([300000])
ax_histy.set_xticklabels([300000], fontsize=14)

ax_histx.set_xlim(ax_scatter.get_xlim())
ax_histy.set_ylim(ax_scatter.get_ylim())

outfile = os.path.join(results_path, 'pre-post_activation_relu_trained.pdf')
fig.savefig(outfile, format='pdf', dpi=200, bbox_inches='tight')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14110999303351654, 'fraction_numerical': 0.046288412415821656, 'mean_word_length': 4.627177700348432, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '398554', 'license': 'MIT', 'n_tokens_mistral': 5233, 'n_tokens_neox': 4846, 'n_words': 813, 'path': 'code/task1_step6_plot_1stlayer_activation_comparison.ipynb', 'repo_name': 'p-koo/exponential_activations', 'size': '12919'}