<jupyter_start><jupyter_text>Validation/Verification & Performance comparison for MNIST digit classification with multiple hidden layers and multiple neurons  Run the following for Google colab then restart runtime<jupyter_code>! pip install --upgrade --no-cache-dir https://github.com/manassharma07/crysx_nn/tarball/main
! pip install IPython==7.7.0<jupyter_output>Collecting https://github.com/manassharma07/crysx_nn/tarball/main
  Downloading https://github.com/manassharma07/crysx_nn/tarball/main
[K     / 8.2 MB 1.5 MB/s
[?25hCollecting numba>=0.54.1
  Downloading numba-0.54.1-cp37-cp37m-manylinux2014_x86_64.manylinux_2_17_x86_64.whl (3.3 MB)
[K     |████████████████████████████████| 3.3 MB 5.2 MB/s 
[?25hCollecting numpy==1.19.2
  Downloading numpy-1.19.2-cp37-cp37m-manylinux2010_x86_64.whl (14.5 MB)
[K     |████████████████████████████████| 14.5 MB 41.8 MB/s 
[?25hRequirement already satisfied: autograd in /usr/local/lib/python3.7/dist-packages (from crysx-nn==0.1.6) (1.3)
Requirement already satisfied: tqdm in /usr/local/lib/python3.7/dist-packages (from crysx-nn==0.1.6) (4.62.3)
Requirement already satisfied: opt_einsum in /usr/local/lib/python3.7/dist-packages (from crysx-nn==0.1.6) (3.3.0)
Collecting nnv
  Downloading nnv-0.0.5-py3-none-any.whl (4.9 kB)
Requirement already satisfied: matplotlib in /usr/local/lib/python3.7/dist-packag[...]<jupyter_text>GPU only  Import stuff<jupyter_code>import numpy as np
print('numpy',np.__version__)

import cupy as cp
print('cupy',cp.__version__)

import matplotlib.pyplot as plt


# Import loss functions
from crysx_nn.loss import MSE_loss_cupy, MSE_loss_grad_cupy
from crysx_nn.loss import BCE_loss_cupy, BCE_loss_grad_cupy
# Import activation functions
from crysx_nn.activation import Sigmoid_cupy, Sigmoid_grad_cupy
from crysx_nn.activation import ReLU_cupy, ReLU_grad_cupy
from crysx_nn.activation import Softmax_cupy, Softmax_grad_cupy
from crysx_nn.activation import Tanh_offset, Tanh_offset
# Import neural network functions
from crysx_nn.network import back_propagation_cupy
from crysx_nn.network import nn_optimize_cupy, nn_optimize_fast_cupy
from crysx_nn.network import forward_feed_cupy
from crysx_nn.network import visualize


from tqdm.notebook import tqdm


# import numpy as np
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras import optimizers
from tensorflow.keras import backend as K
print('tf',tf.__version__)

import torch, torchvision
print('torch',torch.__version__)
rng = np.random.default_rng()
import time
from timeit import default_timer as timer<jupyter_output>numpy 1.19.2
cupy 10.0.0<jupyter_text>Ensure that evertyhing is only using GPU<jupyter_code>torch.cuda.is_available()
print(torch.cuda.current_device())
print(torch.cuda.device(0))
print(torch.cuda.device_count())
print(torch.cuda.get_device_name(0))
print("Num GPUs Available: ", len(tf.config.list_physical_devices('GPU')))
tf.test.is_gpu_available()
print(tf.test.is_built_with_cuda())
# tf.debugging.set_log_device_placement(True)<jupyter_output><empty_output><jupyter_text>Check if NumPy is linked with MKL or not<jupyter_code>np.show_config()<jupyter_output>blas_mkl_info:
    libraries = ['mkl_rt']
    library_dirs = ['C:/Users/manas/anaconda3/envs/crysx_nn\\Library\\lib']
    define_macros = [('SCIPY_MKL_H', None), ('HAVE_CBLAS', None)]
    include_dirs = ['C:/Users/manas/anaconda3/envs/crysx_nn\\Library\\include']
blas_opt_info:
    libraries = ['mkl_rt']
    library_dirs = ['C:/Users/manas/anaconda3/envs/crysx_nn\\Library\\lib']
    define_macros = [('SCIPY_MKL_H', None), ('HAVE_CBLAS', None)]
    include_dirs = ['C:/Users/manas/anaconda3/envs/crysx_nn\\Library\\include']
lapack_mkl_info:
    libraries = ['mkl_rt']
    library_dirs = ['C:/Users/manas/anaconda3/envs/crysx_nn\\Library\\lib']
    define_macros = [('SCIPY_MKL_H', None), ('HAVE_CBLAS', None)]
    include_dirs = ['C:/Users/manas/anaconda3/envs/crysx_nn\\Library\\include']
lapack_opt_info:
    libraries = ['mkl_rt']
    library_dirs = ['C:/Users/manas/anaconda3/envs/crysx_nn\\Library\\lib']
    define_macros = [('SCIPY_MKL_H', None), ('HAVE_CBLAS', None)]
    include_dirs = [[...]<jupyter_text>Start with the common stuff<jupyter_code>nInputs = 784
nOutputs = 10
neurons_per_layer = [4000,4000,10]#[500,500,500,500,1]#[ 100,4,1] #[100,100,100,100,100,100,1]#[100,100,100,100,1]
activation_func_names = ['Sigmoid','Sigmoid', 'Softmax']#['sigmoid', 'sigmoid', 'sigmoid']#['sigmoid', 'sigmoid','sigmoid', 'sigmoid', 'sigmoid', 'sigmoid', 'sigmoid']#['sigmoid', 'sigmoid', 'sigmoid', 'sigmoid', 'sigmoid']
nLayers = len(neurons_per_layer)
eeta = 0.5
nEpochs=10
nBatches = 0
batchSize = 32#nSamples
totSamples = 0<jupyter_output><empty_output><jupyter_text>Activation function dictionary for all the frameworks<jupyter_code>act_func_dict_PyTorch = {'Sigmoid':torch.nn.Sigmoid(),'ReLU':torch.nn.ReLU(),'ELU':torch.nn.ELU(), 'Hardshrink' : torch.nn.Hardshrink(),'Hardsigmoid':torch.nn.Hardsigmoid(),\
                 'Hardtanh':torch.nn.Hardtanh(),'Hardswish':torch.nn.Hardswish(),'LeakyReLU':torch.nn.LeakyReLU(),'LogSigmoid':torch.nn.LogSigmoid(),\
                 'Tanh':torch.nn.Tanh(),'Softmax':torch.nn.Softmax(dim=1),'Softmin':torch.nn.Softmin(dim=1),'LogSoftmax':torch.nn.LogSoftmax()}
act_func_dict_tensorflow = {'Sigmoid':'sigmoid','Softmax':'softmax','ReLU':'relu','LeakyReLU':'leakyrelu'}<jupyter_output><empty_output><jupyter_text>Visualize<jupyter_code>visualize(nInputs, neurons_per_layer, activation_func_names)<jupyter_output><empty_output><jupyter_text>Set inputsUse torchvision to get the dataset, as well as form the input and output matrices for the learning.<jupyter_code>fname = 'MNIST'

### Download and initialize datasets
TrainDS_orig = torchvision.datasets.MNIST(fname, train=True, download=True)
TestDS_orig = torchvision.datasets.MNIST(fname, train=False)<jupyter_output><empty_output><jupyter_text>Plot few examples of the data<jupyter_code>### Plot examples
fig, axes = plt.subplots(nrows=2, ncols=6, figsize=(15., 6.))
for axes_row in axes:
    for ax in axes_row:
        test_index = rng.integers(0, len(TestDS_orig))
        image, orig_label = TestDS_orig[test_index]
        ax.set_axis_off()
        ax.imshow(image)
        ax.set_title('True: %i' % orig_label)<jupyter_output><empty_output><jupyter_text>Currently the dataset is such that each element of `TrainDS_orig` returns an image and the output label(the digit)<jupyter_code>image, orig_label = TrainDS_orig[0]
print(image)
print(orig_label)
plt.imshow(image)<jupyter_output><PIL.Image.Image image mode=L size=28x28 at 0x1C753C1ED48>
5<jupyter_text>We can convert each image in the data to a 28x28 2d-array or a 784x1 1d-array manually as follows<jupyter_code>imgData = np.array(image.getdata()) # Get data values
imgData = imgData/imgData.max() # Renormalize
print(imgData.shape)
imgData = np.reshape(imgData, [28,28]) # Reshape
print(imgData)
print(imgData.shape)<jupyter_output>(784,)
[[0.         0.         0.         0.         0.         0.
  0.         0.         0.         0.         0.         0.
  0.         0.         0.         0.         0.         0.
  0.         0.         0.         0.         0.         0.
  0.         0.         0.         0.        ]
 [0.         0.         0.         0.         0.         0.
  0.         0.         0.         0.         0.         0.
  0.         0.         0.         0.         0.         0.
  0.         0.         0.         0.         0.         0.
  0.         0.         0.         0.        ]
 [0.         0.         0.         0.         0.         0.
  0.         0.         0.         0.         0.         0.
  0.         0.         0.         0.         0.         0.
  0.         0.         0.         0.         0.         0.
  0.         0.         0.         0.        ]
 [0.         0.         0.         0.         0.         0.
  0.         0.         0.         0.         0.         0.
  0.        [...]<jupyter_text>But alternatively, we can just use PyTorch to do it, because we would want to have the same order of batches and everything to keep the benchmark consistent and comparable.Also, instead of the output label being just a digit, it would be better if it is a 10x1 array with each element giving the probability of the classification being a digit in 0-9.<jupyter_code>### Define the label transform from an integer to a set of probabilities
def target_transform(inlabel):
    newlabel = torch.zeros(10)
    newlabel[inlabel] = 1.
    return newlabel

### Reinitialize datasets with the transforms
TrainDS = torchvision.datasets.MNIST(fname, train=True, download=True,
            target_transform=target_transform, transform=torchvision.transforms.ToTensor())
TestDS = torchvision.datasets.MNIST(fname, train=False,
            target_transform=target_transform, transform=torchvision.transforms.ToTensor())

### Initialize DataLoaders as PyTorch convenience
TrainDL = torch.utils.data.DataLoader(TrainDS, shuffle=True, batch_size=batchSize)
TestDL = torch.utils.data.DataLoader(TestDS, batch_size=1000)<jupyter_output><empty_output><jupyter_text>The `TrainDL` dataloader object holds everything we need (as tensors). We would now use it to get the input data and the output data as numpy 2d arrays for our implementation as well as for tf.eras<jupyter_code>totSamples = 0
for inp, labels in TrainDL:
    totSamples = totSamples+1
nBatches = totSamples
print('Total no. of batches: ', totSamples)

totSamples = totSamples*batchSize
print('Total no. of training samples: ', totSamples)<jupyter_output>Total no. of batches:  1875
Total no. of training samples:  60000<jupyter_text>Actually create inputs and outputsCreate total inputs matrix and the total outputs matrix, which hold all the training data<jupyter_code>inputs = np.zeros((totSamples,784))
outputs = np.zeros((totSamples, 10))

i = 0
for inp, labels in TrainDL:
    offset = i*batchSize
    inputs[offset:offset + batchSize,:] = np.reshape(inp, (batchSize,784))
    outputs[offset:offset + batchSize,:] = labels<jupyter_output><empty_output><jupyter_text>Set float32 precision<jupyter_code>inputs = inputs.astype('float32')
outputs = outputs.astype('float32')
print(inputs.dtype)
print(outputs.dtype)<jupyter_output>float32
float32<jupyter_text>Keras implementationIn the following we have to be careful and change the network nodes, and layers manually. TODO: Make the neural network be generated in a loop automatically using the parameters already defined.<jupyter_code># # define the keras model
# model = Sequential()
# model.add(Dense(100, input_dim=2, activation='sigmoid', use_bias=True))
# model.add(Dense(100, activation='sigmoid', use_bias=True))
# model.add(Dense(100, activation='sigmoid', use_bias=True))
# model.add(Dense(100, activation='sigmoid', use_bias=True))
# model.add(Dense(1, activation='sigmoid', use_bias=True))

# model.summary()<jupyter_output><empty_output><jupyter_text>Make the neural network programatically<jupyter_code># define the keras model
model = Sequential()
# First Hidden layer
model.add(Dense(neurons_per_layer[0], input_dim=nInputs, activation=act_func_dict_tensorflow[activation_func_names[0]], use_bias=True))

if nLayers>1:
    # Hidden layers
    for i in range(nLayers-2):
        model.add(Dense(neurons_per_layer[i], activation=act_func_dict_tensorflow[activation_func_names[i]], use_bias=True))
    # Output layer
    model.add(Dense(neurons_per_layer[nLayers-1], activation=act_func_dict_tensorflow[activation_func_names[nLayers-1]], use_bias=True))
    
model.summary()<jupyter_output>Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
dense (Dense)                (None, 4000)              3140000   
_________________________________________________________________
dense_1 (Dense)              (None, 4000)              16004000  
_________________________________________________________________
dense_2 (Dense)              (None, 10)                40010     
=================================================================
Total params: 19,184,010
Trainable params: 19,184,010
Non-trainable params: 0
_________________________________________________________________<jupyter_text>Check the initial weights and biases for each layerNote how the weights matrix is not 4x2 but rather 2x4<jupyter_code>for i in range(nLayers):
    print('\n Weights for layer ',i+1)
    print(model.layers[i].get_weights()[0])
    print('\n Biases for layer ',i+1)
    print(model.layers[i].get_weights()[1])
# model.layers[0].get_biases()<jupyter_output>Weights for layer  1
[[ 0.00759517 -0.02750433 -0.0315942  ...  0.01050172  0.00548506
   0.03047852]
 [-0.01758     0.00141106  0.01138161 ...  0.02496434 -0.02750086
   0.02730082]
 [ 0.01255432  0.00706807  0.03326616 ...  0.00320026 -0.01971038
   0.01656387]
 ...
 [ 0.01120283  0.03465804  0.02238071 ... -0.01295293  0.00383312
  -0.01577294]
 [ 0.00081099  0.02882988 -0.00898308 ... -0.02441491 -0.00773615
  -0.03306478]
 [ 0.00139764  0.00708669  0.02183889 ...  0.01018504 -0.01560521
  -0.03139113]]

 Biases for layer  1
[0. 0. 0. ... 0. 0. 0.]

 Weights for layer  2
[[ 0.02233783  0.00332657  0.00064109 ... -0.01227711 -0.009206
   0.02282103]
 [ 0.02729319 -0.00387556  0.02447082 ... -0.0080578   0.01637241
   0.01112233]
 [ 0.007343    0.01846067 -0.01082897 ... -0.00424582  0.00452366
   0.01338203]
 ...
 [-0.00013262 -0.00447785  0.02512824 ... -0.00099862  0.00369557
  -0.00653114]
 [ 0.01536768 -0.00032201  0.02204894 ... -0.02336465 -0.0063715
  -0.02320035]
 [-0.0150[...]<jupyter_text>Save the weights generated by tf.keras and use them for our implementation as well<jupyter_code>weightsOriginal = []
biasesOriginal = []


for i in range(nLayers):
    weightsOriginal.append(model.layers[i].get_weights()[0].T)
    biasesOriginal.append(model.layers[i].get_weights()[1].T)

print('Weights matrices: ',weightsOriginal)
print('Biases: ',biasesOriginal)<jupyter_output>Weights matrices:  [array([[ 0.00759517, -0.01758   ,  0.01255432, ...,  0.01120283,
         0.00081099,  0.00139764],
       [-0.02750433,  0.00141106,  0.00706807, ...,  0.03465804,
         0.02882988,  0.00708669],
       [-0.0315942 ,  0.01138161,  0.03326616, ...,  0.02238071,
        -0.00898308,  0.02183889],
       ...,
       [ 0.01050172,  0.02496434,  0.00320026, ..., -0.01295293,
        -0.02441491,  0.01018504],
       [ 0.00548506, -0.02750086, -0.01971038, ...,  0.00383312,
        -0.00773615, -0.01560521],
       [ 0.03047852,  0.02730082,  0.01656387, ..., -0.01577294,
        -0.03306478, -0.03139113]], dtype=float32), array([[ 0.02233783,  0.02729319,  0.007343  , ..., -0.00013262,
         0.01536768, -0.01501134],
       [ 0.00332657, -0.00387556,  0.01846067, ..., -0.00447785,
        -0.00032201,  0.01927898],
       [ 0.00064109,  0.02447082, -0.01082897, ...,  0.02512824,
         0.02204894,  0.00361957],
       ...,
       [-0.01227711, -0.0080578 , -0.00[...]<jupyter_text>Compile model<jupyter_code># compile the keras model

# In the following manner we can't set the learning rate of the optimizer
# model.compile(loss='mse', optimizer='sgd', metrics=['mse'])

# So use the following instead
# model.compile(loss='mse', optimizer=optimizers.SGD(learning_rate=eeta), metrics=['mse'])
model.compile(loss='binary_crossentropy', optimizer=optimizers.SGD(learning_rate=eeta), metrics=['binary_crossentropy'])
# model.compile(loss='categorical_crossentropy', optimizer=optimizers.SGD(learning_rate=eeta), metrics=['categorical_crossentropy'])<jupyter_output><empty_output><jupyter_text>Forward feed<jupyter_code>model.evaluate(inputs, outputs, batch_size=batchSize)<jupyter_output>1875/1875 [==============================] - ETA: 0s - loss: 0.1123 - binary_crossentropy: 0.112 - 2s 1ms/step - loss: 0.1123 - binary_crossentropy: 0.1123<jupyter_text>Does tf.model.evaluate change the weights and biases?<jupyter_code>for i in range(nLayers):
    print('\n Weights for layer ',i+1)
    print(model.layers[i].get_weights()[0])
    print('\n Biases for layer ',i+1)
    print(model.layers[i].get_weights()[1])
# model.layers[0].get_biases()<jupyter_output>Weights for layer  1
[[ 0.02770528  0.01342276 -0.02154688 ... -0.01078052  0.02781676
   0.01520592]
 [-0.04179625  0.04243159 -0.00513438 ... -0.05598988  0.01234204
  -0.00272761]
 [-0.02903803  0.03301439 -0.05664266 ...  0.00260885 -0.02461923
   0.05512793]
 ...
 [ 0.02721892 -0.05365229 -0.01171876 ...  0.03809226  0.00783075
   0.04822795]
 [ 0.04791708  0.00905671  0.01673576 ... -0.01449444 -0.0516709
  -0.03389909]
 [-0.01616792  0.04098896  0.05674877 ...  0.0057726   0.01779502
  -0.00874165]]

 Biases for layer  1
[0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0. 0.
 0. 0. 0. 0. 0. 0. 0. 0. 0[...]<jupyter_text>From the above, we can be sure that it does not change the weights and biases  Fit 1 epoch  (forward feed, backpropagation, updating the weights, biases)Let us just try to see and compare the error after just 1 epoch. We should expect the model to perform forward feed, calculate loss/error,perform backpropagation,and adjust the weights and biases based on the learning rate.Note: Batch size=32 indicates that we are using all the training data(examples) in batches.References: https://machinelearningmastery.com/how-to-control-the-speed-and-stability-of-training-neural-networks-with-gradient-descent-batch-size/<jupyter_code># fit the keras model on the dataset
model.fit(inputs, outputs, epochs=1, batch_size=batchSize)<jupyter_output>1875/1875 [==============================] - 7s 4ms/step - loss: 0.1055 - binary_crossentropy: 0.1055<jupyter_text>Now the weights and biases must have been updated<jupyter_code>for i in range(nLayers):
    print('\n Weights for layer ',i+1)
    print(model.layers[i].get_weights()[0])
    print('\n Biases for layer ',i+1)
    print(model.layers[i].get_weights()[1])
# model.layers[0].get_biases()<jupyter_output>Weights for layer  1
[[ 0.02770528  0.01342276 -0.02154688 ... -0.01078052  0.02781676
   0.01520592]
 [-0.04179625  0.04243159 -0.00513438 ... -0.05598988  0.01234204
  -0.00272761]
 [-0.02903803  0.03301439 -0.05664266 ...  0.00260885 -0.02461923
   0.05512793]
 ...
 [ 0.02721892 -0.05365229 -0.01171876 ...  0.03809226  0.00783075
   0.04822795]
 [ 0.04791708  0.00905671  0.01673576 ... -0.01449444 -0.0516709
  -0.03389909]
 [-0.01616792  0.04098896  0.05674877 ...  0.0057726   0.01779502
  -0.00874165]]

 Biases for layer  1
[ 1.71175179e-05  4.02128426e-05 -5.57732938e-06 -4.81689494e-05
 -3.27274065e-05  8.67037488e-06  1.03481816e-05 -9.36488959e-06
 -1.17179179e-05  3.87333130e-05 -2.05888064e-05  4.05125002e-06
  1.77234178e-05  3.96342912e-05  2.74311765e-06 -1.61538410e-05
 -5.80562155e-05  2.04519456e-05 -3.37439315e-06 -1.69575796e-05
  6.36690402e-07  2.27979654e-05  1.00082025e-05 -2.73508749e-06
 -4.62186908e-05 -1.81942032e-05 -3.12332486e-05  2.74778977e-05
 -5.10884[...]<jupyter_text>Now let us do a forward feed again and calculate the loss/error<jupyter_code>out = model.evaluate(inputs, outputs, batch_size=batchSize)
print(out)<jupyter_output>1875/1875 [==============================] - 2s 1ms/step - loss: 0.1055 - binary_crossentropy: 0.1055
[0.10547860711812973, 0.10547860711812973]<jupyter_text>The above result, compares well with the result from my own implementation (when biases are updated independently)  Now let us let the model train for multiple epochs<jupyter_code>%%time
# fit the keras model on the dataset
# history = model.fit(tf.convert_to_tensor(inputs, dtype=tf.float32), tf.convert_to_tensor(outputs, dtype=tf.float32), epochs=nEpochs, batch_size=batchSizenSamples, verbose=0)
history = model.fit(inputs, outputs, epochs=nEpochs, batch_size=batchSize, verbose=0)<jupyter_output>CPU times: user 1min 3s, sys: 5.34 s, total: 1min 9s
Wall time: 1min 2s<jupyter_text>Turns out that this was quite slow. At first, I thought that the problem was that it was printing at each epoch. So I set the verbose=0 (silent). But still, it was incredibly slow compared to my implementation as well as PyTorch, even though we used own for loop in both the cases.<jupyter_code>history.history<jupyter_output><empty_output><jupyter_text>Now also let us have a look at the predictions for the sake of the tutorial<jupyter_code># make probability predictions with the model
predictions = model.predict(inputs)
print(predictions)<jupyter_output>[[0.12565263 0.06827167 0.10207968 ... 0.05959755 0.05968821 0.00693596]
 [0.13112    0.06996221 0.10292286 ... 0.06071939 0.05929473 0.00686394]
 [0.12356264 0.07003288 0.10055003 ... 0.06172275 0.06019304 0.00685642]
 ...
 [0.1271715  0.0720456  0.10047754 ... 0.0600388  0.05858634 0.00705503]
 [0.1271715  0.0720456  0.10047754 ... 0.0600388  0.05858634 0.00705503]
 [0.1271715  0.0720456  0.10047754 ... 0.0600388  0.05858634 0.00705503]]<jupyter_text>We have seen how to initialize custom weights/biases, perform forward feed, train/optimize the model, and finally how to check the updated parameters. Let us now see how we can see the input/output at each layer for debugging purposes.<jupyter_code>for i in range(nLayers):
    print('\n Inputs for layer ',i+1)
    print(model.layers[i].input)
    print('\n Outputs for layer ',i+1)
    print(model.layers[i].output)

for i in range(nLayers):
    print('\n Inputs for layer ',i+1)
    print(model.layers[i].input)
    print('\n Outputs for layer ',i+1)
    print(model.layers[i].output)<jupyter_output>Inputs for layer  1
KerasTensor(type_spec=TensorSpec(shape=(None, 784), dtype=tf.float32, name='dense_input'), name='dense_input', description="created by layer 'dense_input'")

 Outputs for layer  1
KerasTensor(type_spec=TensorSpec(shape=(None, 1000), dtype=tf.float32, name=None), name='dense/Sigmoid:0', description="created by layer 'dense'")

 Inputs for layer  2
KerasTensor(type_spec=TensorSpec(shape=(None, 1000), dtype=tf.float32, name=None), name='dense/Sigmoid:0', description="created by layer 'dense'")

 Outputs for layer  2
KerasTensor(type_spec=TensorSpec(shape=(None, 1000), dtype=tf.float32, name=None), name='dense_1/Sigmoid:0', description="created by layer 'dense_1'")

 Inputs for layer  3
KerasTensor(type_spec=TensorSpec(shape=(None, 1000), dtype=tf.float32, name=None), name='dense_1/Sigmoid:0', description="created by layer 'dense_1'")

 Outputs for layer  3
KerasTensor(type_spec=TensorSpec(shape=(None, 10), dtype=tf.float32, name=None), name='dense_2/Softmax:0', descr[...]<jupyter_text>We can't really see any numbers in the above output So we need to do something more<jupyter_code>for i in range(nLayers):
    print('\n Inputs for layer ',i+1)
    func = K.function([model.get_layer(index=0).input], model.get_layer(index=i).input)
    layerInput = func([inputs])  # input_data is a numpy array
    print(layerInput)
    print('\n Outputs for layer ',i+1)
    func = K.function([model.get_layer(index=0).input], model.get_layer(index=i).output)
    layerOutput = func([inputs])  # input_data is a numpy array
    print(layerOutput)



# Plot the error vs epochs
plt.plot(history.history['loss'])
plt.yscale('log')
plt.show()<jupyter_output><empty_output><jupyter_text>My implementation  Get the cupy versions of input, output, weights and biases arrays<jupyter_code>inputs_cupy = cp.asarray(inputs)
outputs_cupy = cp.asarray(outputs)
weightsOriginal_cupy = []
biasesOriginal_cupy = []
for i in range(nLayers):
    weightsOriginal_cupy.append(cp.asarray(weightsOriginal[i]))
    biasesOriginal_cupy.append(cp.asarray(biasesOriginal[i]))<jupyter_output><empty_output><jupyter_text>Forwad feed once<jupyter_code># Perform Forward feed and get the outputs at each layers and the inputs at each layer
a, z = forward_feed_cupy(inputs_cupy, nLayers, weightsOriginal_cupy, biasesOriginal_cupy, activation_func_names)
print('Outputs at each layer (including input layer):\n ',a)
print('Inputs at each layer :\n ', z)

# Error
print('Error:\n', MSE_loss_cupy(a[nLayers],outputs_cupy)/batchSize)<jupyter_output>Outputs at each layer (including input layer):
  [array([[0., 0., 0., ..., 0., 0., 0.],
       [0., 0., 0., ..., 0., 0., 0.],
       [0., 0., 0., ..., 0., 0., 0.],
       ...,
       [0., 0., 0., ..., 0., 0., 0.],
       [0., 0., 0., ..., 0., 0., 0.],
       [0., 0., 0., ..., 0., 0., 0.]], dtype=float32), array([[0.4441313 , 0.4515828 , 0.5100947 , ..., 0.34285173, 0.45934904,
        0.4371704 ],
       [0.6100375 , 0.52175194, 0.6501559 , ..., 0.44281343, 0.39663258,
        0.52286524],
       [0.6085448 , 0.47681525, 0.40954524, ..., 0.46594366, 0.53735226,
        0.5673876 ],
       ...,
       [0.5       , 0.5       , 0.5       , ..., 0.5       , 0.5       ,
        0.5       ],
       [0.5       , 0.5       , 0.5       , ..., 0.5       , 0.5       ,
        0.5       ],
       [0.5       , 0.5       , 0.5       , ..., 0.5       , 0.5       ,
        0.5       ]], dtype=float32), array([[0.3667317 , 0.69518864, 0.53528893, ..., 0.653972  , 0.2972663 ,
        0.38338184],
      [...]<jupyter_text>Backpropagation<jupyter_code># Get the derivative of the output cost function wrt to the output vector of the output layer
# The input arguments should always be an array
dc_daL = MSE_loss_grad_cupy(a[nLayers], outputs_cupy)/batchSize
print('Gradient of the error function wrt to the predicted output of the output layer\n',dc_daL)

# Perform Back Propagation and get the derivatives wrt the weights and biases
derWeights, derBiases, weights, biases = back_propagation_cupy(z, a, activation_func_names, nLayers, batchSize, weightsOriginal_cupy, biasesOriginal_cupy, eeta, dc_daL)
print('Gradient of Cost (error function) wrt weights:\n',derWeights)
print('Gradient of Cost (error function) wrt biases:\n',derBiases)<jupyter_output>Gradient of the error function wrt to the predicted output of the output layer
 [[ 0.00091932 -0.00500191  0.00021803 ...  0.00030687  0.00038559
   0.00036766]
 [ 0.00094541  0.00123714  0.00020881 ...  0.00029597  0.00037686
   0.00037528]
 [ 0.00094999  0.00124562  0.00020885 ... -0.00593455  0.00039085
   0.00037345]
 ...
 [ 0.000936    0.00123757  0.00021187 ...  0.00030205  0.00038689
   0.00037649]
 [ 0.000936    0.00123757  0.00021187 ...  0.00030205  0.00038689
   0.00037649]
 [ 0.000936    0.00123757  0.00021187 ...  0.00030205  0.00038689
   0.00037649]]
Gradient of Cost (error function) wrt weights:
 [array([[0., 0., 0., ..., 0., 0., 0.],
       [0., 0., 0., ..., 0., 0., 0.],
       [0., 0., 0., ..., 0., 0., 0.],
       ...,
       [0., 0., 0., ..., 0., 0., 0.],
       [0., 0., 0., ..., 0., 0., 0.],
       [0., 0., 0., ..., 0., 0., 0.]], dtype=float32), array([[ 0.01999421,  0.01999554,  0.01999485, ...,  0.01999344,
         0.01998736,  0.01999739],
       [ 0.05209812,  [...]<jupyter_text>Gradient descent (update the weights and biases)<jupyter_code>newWeights = [weightsOriginal_cupy[i] - eeta*derWeights[i] for i in range(nLayers)]
print('New Weights: ', newWeights)

newBiases = [biasesOriginal_cupy[i] - eeta*derBiases[i] for i in range(nLayers)]
# newBiases = biasesOriginal
print('New Biases: ', newBiases)<jupyter_output>New Weights:  [array([[ 0.05362193, -0.04384925, -0.05041252, ...,  0.02452865,
        -0.00975306,  0.05169901],
       [-0.01931749, -0.05192047,  0.042855  , ...,  0.055671  ,
        -0.05218548, -0.0023516 ],
       [ 0.02721496,  0.01917209, -0.04379744, ..., -0.05222633,
        -0.01789423,  0.05113811],
       ...,
       [ 0.03652013, -0.00522443,  0.0027399 , ...,  0.03056527,
         0.01154899,  0.03648541],
       [ 0.03028806,  0.01630289, -0.03326103, ..., -0.04739064,
        -0.03625706,  0.00658271],
       [ 0.00085687,  0.01537386,  0.04452201, ..., -0.04252154,
        -0.05796155,  0.05479489]], dtype=float32), array([[ 0.04445763,  0.0421905 , -0.02465371, ...,  0.01625276,
         0.03802508, -0.01404194],
       [ 0.01457009, -0.06467856, -0.06601943, ..., -0.05402459,
         0.02483728, -0.03198418],
       [ 0.02879   ,  0.03292474,  0.06580479, ...,  0.05197012,
         0.0641099 ,  0.03881461],
       ...,
       [-0.01929006,  0.05277121,  0.0023547[...]<jupyter_text>Forward feed again and check the loss/error<jupyter_code># Perform Forward feed and get the outputs at each layers and the inputs at each layer
a, z = forward_feed_cupy(inputs_cupy, nLayers, weightsOriginal_cupy, biasesOriginal_cupy, activation_func_names)
print('Outputs at each layer (including input layer):\n ',a)
print('Inputs at each layer :\n ', z)

# New Error
print('New Error:\n', MSE_loss_cupy(a[nLayers],outputs_cupy)/batchSize)<jupyter_output>Outputs at each layer (including input layer):
  [array([[0., 0., 0., ..., 0., 0., 0.],
       [0., 0., 0., ..., 0., 0., 0.],
       [0., 0., 0., ..., 0., 0., 0.],
       ...,
       [0., 0., 0., ..., 0., 0., 0.],
       [0., 0., 0., ..., 0., 0., 0.],
       [0., 0., 0., ..., 0., 0., 0.]], dtype=float32), array([[0.4441313 , 0.4515828 , 0.5100947 , ..., 0.34285173, 0.45934904,
        0.4371704 ],
       [0.6100375 , 0.52175194, 0.6501559 , ..., 0.44281343, 0.39663258,
        0.52286524],
       [0.6085448 , 0.47681525, 0.40954524, ..., 0.46594366, 0.53735226,
        0.5673876 ],
       ...,
       [0.5       , 0.5       , 0.5       , ..., 0.5       , 0.5       ,
        0.5       ],
       [0.5       , 0.5       , 0.5       , ..., 0.5       , 0.5       ,
        0.5       ],
       [0.5       , 0.5       , 0.5       , ..., 0.5       , 0.5       ,
        0.5       ]], dtype=float32), array([[0.3667317 , 0.69518864, 0.53528893, ..., 0.653972  , 0.2972663 ,
        0.38338184],
      [...]<jupyter_text>Optimize network  ** With cp.einsum ( Slower than opt_einsum expression reuse)**<jupyter_code>%%time

# Run optimization
from crysx_nn.loss import MSE_loss_cupy, MSE_loss_grad_cupy
optWeights, optBiases, errorPlot = nn_optimize_cupy(inputs_cupy, outputs_cupy, activation_func_names, nLayers, nEpochs=nEpochs, batchSize=batchSize, eeta=eeta, weights=weightsOriginal_cupy, biases=biasesOriginal_cupy, errorFunc=BCE_loss_cupy, gradErrorFunc=BCE_loss_grad_cupy,miniterEpoch=1,batchProgressBar=False,miniterBatch=100)


print(errorPlot)
print(errorPlot)<jupyter_output>[array(0.10553352, dtype=float32), array(0.10548715, dtype=float32), array(0.10548674, dtype=float32), array(0.10548636, dtype=float32), array(0.10548595, dtype=float32), array(0.10548554, dtype=float32), array(0.10548516, dtype=float32), array(0.10548477, dtype=float32), array(0.10548438, dtype=float32), array(0.105484, dtype=float32)]<jupyter_text>Test the quality of the weights by running a forward feed again<jupyter_code># Plot the error vs epochs
plt.plot(errorPlot)
plt.yscale('log')
plt.show()
# Optimized weights and biases
print('Optimized Weights: ', optWeights)
print('Optimized Biases: ', optBiases)
# Forward feed with optimized weights
# Perform Forward feed and get the outputs at each layers and the inputs at each layer
a, z = forward_feed_cupy(inputs_cupy, nLayers, optWeights, optBiases, activation_func_names)
print('Outputs at each layer (including input layer):\n ',a)
print('Inputs at each layer :\n ', z)
# New Error
print('New Average Error with optimized weights:\n', BCE_loss_cupy(a[nLayers],outputs_cupy)/batchSize)
print('Input: ',inputs_cupy)
print('Output: ',a[nLayers])
print('Expected Output: ',outputs_cupy)
print('Difference b/w our weights & biases and keras results:')
for i in range(nLayers):
    print('\nLayer : ',i+1)
    print(abs(cp.asnumpy(optWeights[i])-model.layers[i].get_weights()[0].T).max())
    print(abs(cp.asnumpy(optBiases[i])-model.layers[i].get_weights()[1].T).max())<jupyter_output>Difference b/w our weights & biases and keras results:

Layer :  1
0.0010351948
0.00014224106

Layer :  2
0.0008145459
0.00036059643

Layer :  3
0.005061321
0.004974248<jupyter_text>** NEW METHOD ** With opt_einsum expressions (It is faster than cp.einsum but only slightly)<jupyter_code>%%time

# Run optimization
optWeights, optBiases, errorPlot = nn_optimize_fast_cupy(inputs_cupy, outputs_cupy, activation_func_names, nLayers, nEpochs=nEpochs, batchSize=batchSize, eeta=eeta, weights=weightsOriginal_cupy, biases=biasesOriginal_cupy, errorFunc=BCE_loss_cupy, gradErrorFunc=BCE_loss_grad_cupy,miniterEpoch=1,batchProgressBar=False,miniterBatch=100)


# Plot the error vs epochs
plt.plot(errorPlot)
plt.yscale('log')
plt.show()
# Optimized weights and biases
print('Optimized Weights: ', optWeights)
print('Optimized Biases: ', optBiases)
# Forward feed with optimized weights
# Perform Forward feed and get the outputs at each layers and the inputs at each layer
a, z = forward_feed_cupy(inputs_cupy, nLayers, optWeights, optBiases, activation_func_names)
print('Outputs at each layer (including input layer):\n ',a)
print('Inputs at each layer :\n ', z)
# New Error
print('New Average Error with optimized weights:\n', BCE_loss_cupy(a[nLayers],outputs_cupy)/batchSize)
print('Input: ',inputs_cupy)
print('Output: ',a[nLayers])
print('Expected Output: ',outputs_cupy)
print('Difference b/w our weights & biases and keras results:')
for i in range(nLayers):
    print('\nLayer : ',i+1)
    print(abs(cp.asnumpy(optWeights[i])-model.layers[i].get_weights()[0].T).max())
    print(abs(cp.asnumpy(optBiases[i])-model.layers[i].get_weights()[1].T).max())<jupyter_output>Difference b/w our weights & biases and keras results:

Layer :  1
0.0010351948
0.00014224106

Layer :  2
0.0008145459
0.00036059643

Layer :  3
0.005061321
0.004974248<jupyter_text>PyTorch<jupyter_code>### Choose device: 'cuda' or 'cpu'
# device = 'cpu'
device = 'cuda'<jupyter_output><empty_output><jupyter_text>Generate the network programatically  Method 1<jupyter_code>Layers = []
# Input layer
Layers.append(torch.nn.Linear(nInputs, neurons_per_layer[0]))
Layers.append(torch.nn.Sigmoid())

if nLayers>1:
    # Hidden layers
    for i in range(1,nLayers-1):
        Layers.append(torch.nn.Linear(neurons_per_layer[i-1], neurons_per_layer[i]))
        Layers.append(torch.nn.Sigmoid())
    # Output layer
    Layers.append(torch.nn.Linear(neurons_per_layer[nLayers-2], neurons_per_layer[nLayers-1]))
    Layers.append(torch.nn.Sigmoid())
Network = torch.nn.Sequential(*Layers)
print(Network)<jupyter_output>Sequential(
  (0): Linear(in_features=784, out_features=1000, bias=True)
  (1): Sigmoid()
  (2): Linear(in_features=1000, out_features=1000, bias=True)
  (3): Sigmoid()
  (4): Linear(in_features=1000, out_features=10, bias=True)
  (5): Sigmoid()
)<jupyter_text>Method 2 (Suitable for us to modify weights for each layer to be same as keras and my implementation)<jupyter_code>Network = torch.nn.Sequential()
 
# Input layer -> 1st hidden layer
Network.add_module(str(0),torch.nn.Linear(nInputs, neurons_per_layer[0]))
# Activation function after first hidden layer
Network.add_module('act0',act_func_dict_PyTorch[activation_func_names[0]])

if nLayers>1:
    # Hidden layers
    for i in range(1,nLayers-1):
        Network.add_module(str(i),torch.nn.Linear(neurons_per_layer[i-1], neurons_per_layer[i]))
        Network.add_module('act'+str(i),act_func_dict_PyTorch[activation_func_names[i]])
        
    # Output layer
    Network.add_module(str(nLayers-1),torch.nn.Linear(neurons_per_layer[nLayers-2], neurons_per_layer[nLayers-1]))
    Network.add_module('act'+str(nLayers-1),act_func_dict_PyTorch[activation_func_names[nLayers-1]])
# torch.nn.init.constant_(Network.weight, 2.0)
Network.to(device=device)
print(next(Network.parameters()).is_cuda)<jupyter_output>False<jupyter_text>initial weights and biases generated by Pytorch<jupyter_code>print(Network[0].weight)
print(Network[0].bias)<jupyter_output>Parameter containing:
tensor([[ 0.0034,  0.0172,  0.0026,  ..., -0.0328, -0.0265, -0.0141],
        [-0.0256, -0.0263, -0.0235,  ..., -0.0343, -0.0344,  0.0115],
        [ 0.0055,  0.0165,  0.0050,  ...,  0.0325, -0.0254, -0.0199],
        ...,
        [-0.0040,  0.0295,  0.0058,  ..., -0.0271,  0.0019,  0.0176],
        [ 0.0257,  0.0084, -0.0091,  ...,  0.0221, -0.0227,  0.0131],
        [-0.0033, -0.0069,  0.0023,  ...,  0.0109,  0.0106,  0.0216]],
       requires_grad=True)
Parameter containing:
tensor([-2.2856e-02,  8.5836e-03, -9.5982e-03,  2.9417e-02, -4.4941e-03,
         1.8952e-02, -3.1176e-02,  2.1700e-02,  1.4473e-02,  1.1495e-02,
        -1.2904e-02, -2.5171e-02, -2.2608e-02, -1.9122e-02, -3.1597e-02,
         9.8738e-03, -2.2397e-02, -1.2338e-02, -2.1534e-02, -1.4771e-02,
         1.7786e-02,  6.9671e-03,  2.6727e-02,  3.9797e-03, -4.8005e-03,
        -3.3332e-02, -1.4991e-03,  6.5190e-03, -3.3893e-02, -2.0609e-02,
         9.6864e-03,  2.9717e-02,  3.0058e-02,  2.0188e-0[...]<jupyter_text>Change them to be the same as others **IMPORTANT NOTE**: The returned tensor and the original array share the same memory. So any changes to the tensor change the array as well.Reference: https://pytorch.org/docs/stable/generated/torch.from_numpy.html<jupyter_code>for i in range(nLayers):
    # Weights
#     Network._modules[str(i)].weight.data = torch.from_numpy(weightsOriginal[i]).to(dtype=torch.float32)
    Network._modules[str(i)].weight.data = torch.tensor(weightsOriginal[i]).to(dtype=torch.float32)
    # Biases
#     Network._modules[str(i)].bias.data = torch.from_numpy(biasesOriginal[i]).to(dtype=torch.float32) # Convert to float32
    Network._modules[str(i)].bias.data = torch.tensor(biasesOriginal[i]).to(dtype=torch.float32) # Convert to float32
for i in range(nLayers):
    # Weights
    print('Weights:\n')
    print(Network._modules[str(i)].weight.data)
    # Biases
    print('Biases:\n')
    print(Network._modules[str(i)].bias.data)
### Get information about model
totpars = 0
for par in Network.parameters():
    newpars = 1
    for num in par.shape:
        newpars *= num
    totpars += newpars
print(Network)
print('%i trainable parameters' % totpars)

### Initialize loss function and optimizer
# crit = torch.nn.MSELoss()
crit = torch.nn.BCELoss().to(device=device)
# crit = torch.nn.CrossEntropyLoss()
opt = torch.optim.SGD(Network.parameters(), lr=eeta)
inputsTorch = torch.from_numpy(inputs)
inputsTorch = inputsTorch.to(dtype=torch.float32) # Convert to float32
# Move to CPU/GPU
inputsTorch = inputsTorch.to(device=device)
labels = torch.from_numpy(outputs)
labels = labels.to(dtype=torch.float32) # Convert to float32
# Move to CPU/GPU
labels = labels.to(device=device)
print(inputsTorch)
print(labels)
%%time
### Set model in training mode and create the epochs axis
Network.train()
epochs = range(1, nEpochs+1)
errorPlot = []

Network.to(device=device)
print(inputsTorch.device)
print(labels.device)
print(next(Network.parameters()).is_cuda)

### Train the model
for e in tqdm(epochs):
    tr_loss = 0.
    samples = 0
    ### Loop over batches
    for iBatch in tqdm(range(nBatches),leave=False):
        offset = iBatch*batchSize
        inputsBatch = inputsTorch[offset:offset + batchSize,:]# Input vector
#            print(x.shape)
        labelsBatch = labels[offset:offset + batchSize,:] # Expected output
        opt.zero_grad() # zero gradient values
        inputsBatch = inputsBatch.to(device=device) # move input and label tensors to the device with the model
        labelsBatch = labelsBatch.to(device=device)   
        outputsTorch = Network(inputsBatch)#.to(device=device) # compute model outputs
        loss = crit(outputsTorch, labelsBatch) # compute batch loss
        loss.backward() # back-propagate the gradients
        opt.step() # update the model weights
        tr_loss += loss.clone().cpu().item()*len(inputsBatch) # add the batch loss to the running loss
        samples += len(inputsBatch) # update the number of processed samples
    tr_loss /= samples # compute training loss
    errorPlot.append(tr_loss)
    print(e, tr_loss)
# Plot the error vs epochs
plt.plot(errorPlot)
plt.yscale('log')
plt.show()
print('Difference b/w our weights & biases and PyTorch results:')
for i in range(nLayers):
    print('\nLayer : ',i+1)
    print(abs(torch.tensor(cp.asnumpy(optWeights[i])).to(dtype=torch.float32)-Network._modules[str(i)].weight.data.cpu()).max())
    print(abs(torch.tensor(cp.asnumpy(optBiases[i])).to(dtype=torch.float32)-Network._modules[str(i)].bias.data.cpu()).max())
torch.get_num_threads()
# torch.set_num_threads(4)
torch.get_num_threads()
torch.get_num_threads()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15012716423750366, 'fraction_numerical': 0.1327399002249829, 'mean_word_length': 3.5833893745796908, 'pattern_counts': {'":': 0, '<': 146, '<?xml version=': 0, '>': 151, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 13, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '43', 'dir': 'jupyter-structured-clean-dedup', 'id': '606583', 'license': 'MIT', 'n_tokens_mistral': 18632, 'n_tokens_neox': 15328, 'n_words': 3294, 'path': 'benchmarks_tests/Performance_MNIST_GPU.ipynb', 'repo_name': 'manassharma07/cysx_nn', 'size': '40892'}