<jupyter_start><jupyter_text>Function Definitions<jupyter_code>def z_score(arr, axis = 0):
    '''
        Transform 2d array to zero mean unit variance data along the time axis (2nd dim).
        
    '''
    
    means = np.mean(arr, axis = 0, keepdims = True)
    stds = np.std(arr, axis = 0, keepdims = True)
    
    arr -= means
    arr /= stds
    
    return arr

def evaluate_model(Y_pred, Y_true, metric = 'mse'):
    '''
    Evaluate a set of predictions Y_pred, comparing to Y_true true images. Return the
    average percent error over the set
    '''
    num_samples = Y_pred.shape[0]

    if metric == 'mse':
        pct_errs  = np.divide(np.linalg.norm(Y_pred - Y_true, axis = 1), np.linalg.norm(Y_true, axis = 1))
        
    return np.sum(pct_errs) / num_samples

def get_feedforward_model(params, num_pixels, num_neurons):
    '''
    Generate a feedforward deep neural network that maps neuron input to pixel intensity output. 
    ## TODO comment
    '''   

    (learning_rate, beta_1, beta_2, depth, width, loss) =(
    params['learning_rate'], params['beta_1'], params['beta_2'], params['depth'], params['width'], params['loss']
    )
    
    
    
    inputs = keras.Input(shape = (num_neurons,))
    for j in range(depth):
        if j == 0:
            x = Dense(units = width)(inputs)
        else:
            x = Dense(units = width)(x)
        x = LayerNormalization()(x)

    outputs = Dense(units = num_pixels)(x)

    model = Model(inputs = inputs, outputs = outputs)

    
    
    model.compile(
        optimizer = keras.optimizers.Adam(
            learning_rate = learning_rate,
            beta_1 = beta_1,
            beta_2 = beta_2
        ), 
        loss = loss
    )
    
    return model<jupyter_output><empty_output><jupyter_text>Load Data from Preprocessing<jupyter_code>file_name_spikes = 'spike_rates_width=167.0_ms.npy'
file_name_frames = 'stimulus_frames.npy'

X = z_score(np.load(file_name_spikes))
Y = np.load(file_name_frames)

X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.1, random_state = 1)

num_pixels = Y_train.shape[1]
num_neurons = X_train.shape[1]<jupyter_output><empty_output><jupyter_text>Feedforward Deep Network<jupyter_code>params = {
    'learning_rate' : .001,
    'beta_1' : .99,
    'beta_2' : .999,
    'depth' : 10,
    'width' : 32,
    'loss' : keras.losses.MeanSquaredError()
}

model = get_feedforward_model(params, num_pixels, num_neurons)

#gridsearch learning rate, width, depth
num_epochs  = 1000

# number of learning rate values to explore (log scale)
lr_res = 10  

# number of depths to explore (log scale)
depth_res = 10 


# number of hidden layer widhts to explore
width_res = 10

learning_rates =  np.logspace(-5, -.1, num = lr_res)
depths = np.logspace(0, 2, num = depth_res).astype(np.int)
widths = np.logspace(1, 3, num = 10)


sweep_lr_d = False
if sweep_lr_d:
    performances = np.zeros((lr_res, depth_res))

    for i, lr in enumerate(learning_rates):
        for j, d in enumerate(depths):
            search_params = params.copy()
            search_params['learning_rate'] = lr
            search_params['depth'] = d
            
            model = get_feedforward_model(params, num_pixels, num_neurons)(search_params)
            model.fit(X_train, Y_train, epochs=1000)
            
            y_hat_train = model.predict(X_train)
            performances[i,j] = evaluate_model(y_hat_train, Y_train)

            
            
# Empircal Results so far
best_rate = learning_rates[2]
best_depth = depths[4]
best_width = widths[4]

sweep_w = False
if sweep_w:
    
    performances = np.zeros((width_res,))
    for i, w in enumerate(widths):
        search_params = params.copy()
        search_params['learning_rate'] = best_rate
        search_params['depth'] = best_depth
        search_params['width'] = w
        
        model = get_feedforward_model(params, num_pixels, num_neurons)(search_params)
        model.fit(X_train, Y_train, epochs=1000)

        y_hat_train = model.predict(X_train)
        performances[i] = evaluate_model(y_hat_train, Y_train)

        
train_best_model = False
if train_best_model:
            
    best_params = params.copy()
    best_params['learning_rate'] = best_rate
    best_params['depth'] = best_depth
    best_params['width'] = best_width
    best_params['loss'] = 'mse'

    model = get_feedforward_model(best_params, num_pixels, num_neurons)
    model.fit(X_train, Y_train, epochs = 50000, verbose = 1)
    
    y_hat_train = model.predict(X_train)
    y_hat_test = model.predict(X_test)

    train_error =  evaluate_model(y_hat_train, Y_train)
    test_error = evaluate_model(y_hat_test, Y_test)
    
    print('Feedforward Deep Net Train Error: {0}'.format(train_error))
    print('Feedforward Deep Net Test Error: {0}'.format(test_error))

model.summary()


# Save / Load Existing Trained Model
#model.save('ff_deep_net_trained')
model = keras.models.load_model('ff_deep_net_trained')
# Evaluate Feedforward Deep Net

y_hat_train = model.predict(X_train)
y_hat_test = model.predict(X_test)

train_error =  evaluate_model(y_hat_train, Y_train, metric='ssim')
test_error = evaluate_model(y_hat_test, Y_test, metric='ssim')

print('Feedforward Deep Net Train Error: {0}'.format(train_error))
print('Feedforward Deep Net Test Error: {0}'.format(test_error))


filters = np.linalg.pinv(X_train) @ Y_train
train_error = evaluate_model(X_train @ filters, Y_train, metric='mse')
test_error = evaluate_model(X_test @ filters, Y_test, metric ='mse')

print('Linear Decoder Train Error: {0}'.format(train_error))
print('Linear Decoder Test Error: {0}'.format(test_error))
sample_num = 1
y_hat_test = model.predict(X_test)

plt.figure(figsize=(16,8))
plt.imshow(y_hat_test[sample_num,:].reshape(64, 128), cmap = 'bone')
plt.title('FeedForward Deep Network Image Prediction, Sample Number {0}'.format(sample_num))
plt.show()

plt.figure(figsize=(16,8))
plt.imshow(Y_test[sample_num, :].reshape(64, 128), cmap = 'bone')
plt.title('Presnted Image, Sample Number {0}'.format(sample_num))
plt.show()

plt.figure(figsize=(16,8))
plt.imshow((X_test @ filters)[sample_num, :].reshape(64, 128), cmap = 'bone')
plt.title('Linear Decoder Image Prediction , Sample Number {0}'.format(sample_num))
plt.show()
impulse = np.zeros(X_test.shape)
for i in range(impulse.shape[1]):
    impulse[i,i] = 1

    
y_impulse = model.predict(impulse)
neuron_idx = 512
    
plt.figure(figsize=(16,8))
plt.imshow((y_impulse)[neuron_idx, :].reshape(64, 128), cmap = 'bone')
plt.title('Deep Decoder Response to Neuron {0}'.format(neuron_idx))
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12322561159770462, 'fraction_numerical': 0.018574448807006948, 'mean_word_length': 3.5993055555555555, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '596696', 'license': 'MIT', 'n_tokens_mistral': 2456, 'n_tokens_neox': 2331, 'n_words': 495, 'path': '.ipynb_checkpoints/Deep Network Training-checkpoint.ipynb', 'repo_name': 'Fritz17236/deep_decoding', 'size': '6622'}