<jupyter_start><jupyter_text>Importing Libraries and downloading data<jupyter_code>import numpy as np
import pandas as pd
from tensorflow.keras.datasets import mnist, fashion_mnist
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
import numpy as np
from sklearn.metrics import mean_squared_error<jupyter_output><empty_output><jupyter_text>Question 1 : Prepare dataset from fashion mnist<jupyter_code>def prepare_dataset():
    (X, y), (X_test, y_test) = fashion_mnist.load_data()

    # Reshaping the data matrices
    X = X.reshape(X.shape[0], -1)
    X_test = X_test.reshape(X_test.shape[0], -1)

    # Normalizing the pixel intensities
    X = X/255.0
    X_test = X_test/255.0
    
# splitting the X_train into a training set and validation set
    X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.1, random_state=42)

  # encoding for class labels  
    y_train_encode = np.zeros((10,y_train.shape[0]))
    y_train_encode[y_train, np.array(list(range(y_train.shape[0])))] = 1

    y_val_encode = np.zeros((10,y_val.shape[0]))
    y_val_encode[y_val, np.array(list(range(y_val.shape[0])))] = 1

    
    #print training validation and test set 
    print("Number of examples in training set")
    print(X_train.shape)
    print(y_train_encode.shape)
    print(y_train.shape)
    
    print("No of examples in validation set")
    print(X_val.shape)
    print(y_val.shape)
    print("No of examples in test set")
    print(X_test.shape)
    print(y_test.shape)
    
    return X_train.T, y_train_encode, y_train.T ,X_val.T, y_val.T, X_test.T, y_test.T, y_val_encode

#return all train valid test data 
X_train, y_train_encode, y_train ,X_val, y_val, X_test, y_test,y_val_encode = prepare_dataset()<jupyter_output>Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-labels-idx1-ubyte.gz
32768/29515 [=================================] - 0s 0us/step
40960/29515 [=========================================] - 0s 0us/step
Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-images-idx3-ubyte.gz
26427392/26421880 [==============================] - 0s 0us/step
26435584/26421880 [==============================] - 1s 0us/step
Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-labels-idx1-ubyte.gz
16384/5148 [===============================================================================================] - 0s 0us/step
Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-images-idx3-ubyte.gz
4423680/4422102 [==============================] - 0s 0us/step
4431872/4422102 [==============================] - 0s 0us/step
Number of examples in training set
(54000, 784)
(10, 54[...]<jupyter_text>plot sample image each class in WANDB<jupyter_code>#Number classes 
num_classes = 10
class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',
               'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']
#logging sample image in each class in wandb                
wandb.init(project="test_100", entity="moni6264", name="log_images")
#images is a list containing one sample image per class 
indices = [list(y_train.T).index(i) for i in range(num_classes)]
images = []
captions = []
for index in indices:
    images.append(X_train.T[index].reshape((28, 28)))
    captions.append(class_names[y_train.T[index]])

# Log one sample image of each class to wandb
wandb.log({"Image from each class": [wandb.Image(image, caption=caption) for image, caption in zip(images, captions)]})<jupyter_output><empty_output><jupyter_text>Question.2   Initialize Activation Function Sigmoid,Relu,Tanh,Softmax and derivatives Sigmoid,derivatives Relu and derivatives Tanh<jupyter_code># initialize sigmoid function 
def sigmoid(x):
    return 1. / (1.+np.exp(-x))
# initialize relu function    
def relu(x):
    return np.maximum(0,x)
# initialize tanh function
def tanh(x):
    return 1.*(np.exp(x)+np.exp(-x))/(np.exp(x)-np.exp(-x))
# initialize softmax function    
def softmax(x):
    soft = np.zeros(x.shape)
    for i in range(0, x.shape[1]):
        numr = np.exp(x[:, i])
        soft[:, i] = numr/np.sum(numr)
    return soft


#Derivatives of sigmoid function
def sigmoid_derivative(x):
    return sigmoid(x) * (1-sigmoid(x))
#Derivatives of relu function    
def relu_derivative(x):
    return 1*(x>0)
#Derivatives of tanh function
def tanh_derivative(x):
    return (1 - (np.tanh(x)**2))

#Initialisation of parameters
def initialize_parameters(input_layer, hidden_layer, output_layer,initilisation):
    weights = []
    biases  = []
    #  list of number of neurons per layer specifying layer dimensions in the format [#input_features,#hiddenunits.outputclass]
    layers = [input_layer] + hidden_layer + [output_layer]  
    for i in range(len(hidden_layer)+1):
        if initilisation == 'random': # initilisation mode random
            weights.append(np.random.rand(layers[i+1], layers[i])*0.01)
        if initilisation == 'xavier': # initilisation mode xavier 
            weights.append(np.random.randn(layers[i+1],layers[i])*np.sqrt(2/layers[i+1]))
        biases.append(np.random.randn(layers[i+1],1))
    return weights, biases<jupyter_output><empty_output><jupyter_text>Question 2 and 3 initilisation Feedforward and back propogation, six optimizer function sgd,momentum based gradient descent,nesterov accelerated gradient descent,rmsprop,adam,nadam,<jupyter_code>class ANN:

    def __init__(self,n_inputs,n_hidden,n_outputs,initialisation, activation,loss):
           
        self.input_layer   = n_inputs # input layer
        self.output_layer  = n_outputs # output layer
        self.hidden_layer   = n_hidden # hidden layer
        self.activation = activation # activation function (sigmiod,reli,tahn,softmax)
        self.loss_function = loss # loss (mse entropy)
        self.weights    = []
        self.biases     = []
        self.initialisation = initialisation
        self.weights,self.biases = initialize_parameters(self.input_layer,self.hidden_layer,self.output_layer,self.initialisation)

    
    
    #Forward Propogation code here
    def forward_propagation(X, weights, biases,activation):

        A = []  #pre-activation
       
        H  = []    # post activation
        W      = weights  # weights
        b      = biases   # biases 

        k=0
        A.append(np.dot(W[k],X)+b[k])
        if activation == 'sigmoid':
          H.append(sigmoid(A[k]))
        elif activation == 'tanh':
          H.append(tanh(A[k]))
        elif activation == 'relu':
          H.append(relu(A[k]))


        for k in range(1,len(hidden_layer)):
            A.append(np.dot(W[k],H[k-1])+b[k])
            if activation == 'sigmoid':
              H.append(sigmoid(A[k]))
            elif activation == 'tanh':
              H.append(tanh(A[k]))
            elif activation == 'relu':
              H.append(relu(A[k]))
            elif activation == 'leaky_relu':
              H.append(leaky_relu(A[k]))

        k=len(hidden_layer)
        A.append(np.dot(W[k],H[k-1])+b[k])
        H.append(softmax(A[k]))

        return A , H
        A, H = forward_propagation(X_train, weights, biases, 'sigmoid') # activation function for output layer
        for i in range(len(A)):
          print(A[i].shape)
          for i in range(len(H)):
            print(H[i].shape)
    


     #Back propogation
    def back_propagation(self,X_train,y_train):

        g_a  = [0]*(len(self.hidden_layer)+1)
        g_h  = [0]*(len(self.hidden_layer)+1)
        g_w  = [0]*(len(self.weights))
        g_b  = [0]*(len(self.biases))

        n_samples = X_train.shape[1] 

        for k in reversed(range(len(self.hidden_layer)+1)):
            if k == len(self.hidden_layer):
              if self.loss_function == 'cross_entropy':
                  g_a[k] = self.H[k]  - y_train  
              elif self.loss_function == 'square_loss': 
                  g_a[k] = (self.H[k] - y_train) * self.H[k] * (1 - self.H[k]) 
                
            else:
                g_h[k] = (1/n_samples)*np.matmul(self.weights[k+1].T,g_a[k+1])
                if self.activation == 'sigmoid':
                  g_a[k] = (1/n_samples)*np.multiply(g_h[k],sigmoid_derivative(self.A[k]))
                elif self.activation == 'tanh':
                  g_a[k] = (1/n_samples)*np.multiply(g_h[k],tanh_derivative(self.A[k]))
                elif self.activation == 'relu':
                  g_a[k] = (1/n_samples)*np.multiply(g_h[k],relu_derivative(self.A[k]))
                

            if k == 0:
                g_w[k] = (1/n_samples)*np.matmul(g_a[k],X_train.T) 
            else:
                g_w[k] = (1/n_samples)*np.matmul(g_a[k],self.H[k-1].T)

            g_b[k]  = (1/n_samples)*np.sum(g_a[k], axis=1, keepdims = True)
        return g_w,g_b   # return gradients
    def predict(self, X,y ):
      output =  self.forward_propagation(X)
      out_class=(np.argmax(output,axis=0))
      accuracy = round(accuracy_score(y, out_class) * 100, 3)
      return accuracy
#This block contains functions that implement stochastic gradient descent, momentum, RMSprop, 
#Adam. Nadam and Nestorov accelerated gradient are implemented inside the fit predict function.

    def fit_predict(self,X_train,y_train,X_val ,y_val ,epochs,learning_rate,optimiser='gd',batch_size = 64,lambd=0.0005): 
      steps = 0
      pre_update_w = np.multiply(self.weights,0) # previous update weight
      pre_update_b = np.multiply(self.biases,0) # previous update biases
      update_w = np.multiply(self.weights,0) # update weight 
      update_b = np.multiply(self.biases,0) # update biases
      vw = 0.0 #contains previous W and b values, accumulated in a weighted fashion along with the gradients 
      vb = 0.0 # contains previous W and b values, accumulated in a weighted fashion along with the gradients^2
      eps = 1e-8 # for numerical stability
      a1 =0.0  # for numerical stability
      gamma = 0.9 # for numerical stability
      beta = 0.999 # for numerical stability
      beta1 = 0.9 # for numerical stability
      beta2 = 0.999 # for numerical stability
      m_t, v_t, m_hat_w, v_hat_w, m_b,v_b,m_hat_b,v_hat_b = 0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0 

      for i in range(epochs):
        

            for batch in range(0, X_train.shape[1], batch_size):

              train_b_imag =  X_train[:,batch:batch+batch_size]
              train_l_imag =  y_train_encode[:,batch:batch+batch_size]
              output =  self.forward_propagation(train_b_imag)
              g_weights,g_biases = self.back_propagation(train_b_imag,train_l_imag)

              #gradient descent
              if optimiser == 'gd':
                 self.weights = self.weights - np.multiply(learning_rate,g_weights) - np.multiply(learning_rate*lambd,self.weights)
                 self.biases = self.biases - np.multiply(learning_rate,g_biases)
              #Momemtum gradiend descent code
              if optimiser == 'mgd':
                 
                 update_w = np.multiply(gamma,pre_update_w) + np.multiply(learning_rate,g_weights)
                 self.weights = self.weights - update_w - np.multiply(learning_rate*lambd,self.weights)
                
                 update_b = np.multiply(gamma,pre_update_b) + np.multiply(learning_rate,g_biases)
                 self.biases = self.biases - update_b
                 pre_update_w = update_w
                 pre_update_b = update_b



              #NAG code here
              if optimiser == 'ngd':
                 
                 cache_w = self.weights
                 cache_b = self.biases
                 self.weights = self.weights - np.multiply(gamma,pre_update_w)
                 self.biases = self.biases - np.multiply(gamma,pre_update_b)
                 output =  self.forward_propagation(train_b_imag)
                 g_weights,g_biases = self.back_propagation(train_b_imag,train_l_imag)

                 update_w = np.multiply(gamma,pre_update_w) + np.multiply(learning_rate,g_weights)
                 cache_w = cache_w - update_w - np.multiply(learning_rate*lambd,cache_w)
                    
                 update_b = np.multiply(gamma,pre_update_b) + np.multiply(learning_rate,g_biases)
                 cache_b = cache_b - update_b
                 
                 self.weights = cache_w
                 self.biases = cache_b
                 pre_update_w = update_w
                 pre_update_b = update_b



              #RMS prop code here 
              if optimiser == 'rmsprop': 

                 vw = np.multiply(vw,beta) + np.multiply(1-beta,np.power(g_weights,2))
                 vb = np.multiply(vb,beta) + np.multiply(1-beta,np.power(g_biases,2))

          
                 update_w = np.multiply(g_weights,learning_rate/np.power(vw+eps,1/2))
                 update_b = np.multiply(g_biases,learning_rate/np.power(vb+eps,1/2))
                 
                 self.weights = self.weights - update_w - np.multiply(learning_rate*lambd,self.weights)
                 self.biases = self.biases - update_b
              #Adam code here    
              if optimiser == 'adam':
                 m_t = np.multiply(beta1,m_t) + np.multiply(1-beta1,g_weights)
                 v_t = np.multiply(beta2,v_t) + np.multiply(1-beta2,np.power(g_weights,2))
                 m_b = np.multiply(beta1,m_b) + np.multiply(1-beta1,g_biases)
                 v_b = np.multiply(beta2,v_b) + np.multiply(1-beta2,np.power(g_biases,2))
                
                 m_hat_w = m_t/(1 - np.power(beta1,i+1))
                 m_hat_b = m_b/(1 - np.power(beta1,i+1))
                
                 v_hat_w = v_t/(1 - np.power(beta2,i+1))
                 v_hat_b = v_b/(1 - np.power(beta2,i+1))
                 update_w = (learning_rate / np.power(v_hat_w + eps, 1/2)) * m_hat_w
                 update_b = (learning_rate / np.power(v_hat_b + eps, 1/2)) * m_hat_b
                 self.weights = self.weights - update_w - np.multiply(learning_rate*lambd,self.weights)
                 self.biases = self.biases - update_b
              #Nadam code here
              if optimiser == 'nadam':
                 
                 self.weights = self.weights - np.multiply(gamma,update_w)
                 self.biases  = self.biases  - np.multiply(gamma,update_b)

                 g_weights,g_biases = self.back_propagation(train_b_imag,train_l_imag)

                 m_t =  np.multiply(beta1,m_t) + np.multiply(1 - beta1,g_weights)
                 v_t =  np.multiply(beta2,v_t) + np.multiply(1 - beta2,np.power(g_weights, 2))

                 m_b =  np.multiply(beta1,m_b) + np.multiply(1 - beta1,g_biases)
                 v_b =  np.multiply(beta2,v_b) + np.multiply(1 - beta2,np.power(g_biases, 2))
                
                 m_hat_w = m_t / (1 - np.power(beta1, i+1)) 
                 v_hat_t = v_t / (1 - np.power(beta2, i+1))

                 m_hat_b = m_b / (1 - np.power(beta1, i+1)) 
                 v_hat_b = v_b / (1 - np.power(beta2, i+1))
  
                 a1 = (1-beta1)/(1-np.power(beta,i+1))
                 update_w = np.multiply(learning_rate/(np.power(v_hat_t + eps,1/2)),(np.multiply(a1,g_weights) + np.multiply(beta1,m_hat_w)))#(a4 + a2))
                 update_b = np.multiply(learning_rate/(np.power(v_hat_b + eps,1/2)),(np.multiply(a1,g_biases)+np.multiply(beta1,m_hat_b) ))#(a5 + a3))
                 self.weights = self.weights - update_w - np.multiply(learning_rate*lambd,self.weights)
                 self.biases = self.biases - update_b
            #Training loss for full dataset
            predicted_train = self.forward_propagation(X_train)
            predicted_train_label=(np.argmax(predicted_train,axis=0)) 
            # accuracy  
            acc1 = round(accuracy_score(y_train, predicted_train_label) * 100, 3)

            predicted_val = self.forward_propagation(X_val)
            predicted_val_label=(np.argmax(predicted_val,axis=0))
              
            acc2 = round(accuracy_score(y_val, predicted_val_label) * 100, 3)

            a =self.weights[1:len(self.hidden_layer)]
            b = np.sum([(np.sum((a[i]**2).reshape(1,-1))) for i in range(len(a))]) 
            if self.loss_function == 'cross_entropy':
              train_loss = (-np.sum(np.multiply(y_train_encode,np.log(predicted_train)))+((lambd/2.)*b))/y_train.shape[0]
              val_loss = (-np.sum(np.multiply(y_val_encode,np.log(predicted_val)))+((lambd/2.)*b))/y_val.shape[0]
           #print the training accuracy , training loss , validation accuracy , validation loss  
            print('Epoch {}: training_accuracy = {:.2f} %, training_loss= {:.5f}  ,Validation accuracy = {:.2f} ,Validation loss = {:.5f}'.format(i,acc1,train_loss,acc2,val_loss))
          # login wandb 
            wandb.log({"val_accuracy": acc2,"accuracy": acc1,"steps":epochs,"train_loss":train_loss,"val_loss":val_loss},)

            

         
      return acc1,acc2,train_loss,val_loss<jupyter_output><empty_output><jupyter_text>login into wandb<jupyter_code>%pip install wandb -q
import wandb
wandb.login()<jupyter_output><empty_output><jupyter_text>In this cell, we set up the configurations for the various hyperparameters and use the Sweeps feature to find the combination that gives us best validation accuracy.(WANDB)<jupyter_code>sweep_config = {
    'method': 'random', #grid, random
    'metric': {
      'name': 'val_accuracy',
      'goal': 'maximize'  
    },
    'parameters': {
        'epochs': {
            'values': [5,10,20,40]
        },
        'learning_rate': {
            'values': [1e-2,1e-3]
        },
        'loss_function':{
            'values':['cross_entropy']
        },
        'initilisation':{
            'values':['xavier']
        },
        'batch_size':{
            'values':[32,64,128]
        },
        'optimiser': {
            'values': ['gd','mgd','ngd', 'rmsprop','adam','nadam']
        },
        'activation': {
            'values': ['relu','tanh','sigmoid']
        },
        'hidden_layer': {
            'values': [[32,64,128],
                       [32,32,32],[32,32,32,32],[32,32,32,32,32],[64,64,64],
                       [64,64,64,64],[64,64,64,64,64],[128,128,128],
                       [128,128,128,128],[128,128,128,128,128]]
        },
        
        'weight_decay':{
            'values':[0.0,0.0005]
        }
        
    }
}<jupyter_output><empty_output><jupyter_text>sweep running  ID<jupyter_code>sweep_id = wandb.sweep(sweep_config, entity="moni6264", project="test_100")<jupyter_output><empty_output><jupyter_text>Question 4&5&6<jupyter_code>def train():
    #steps = 10
    # Default values for hyper-parameters we're going to sweep over
    config_defaults = {
        'epochs': 5,
        'input_layer': 28*28,
        'output_layer': 10,
        'batch_size':64,
        'weight_decay':0.005,
        'learning_rate': 1e-3,
        'hidden_layer':[16,16,16],
        'learning_rate':1e-2,
        'optimiser':'mgd',
        'activation':'sigmoid',
        'initialisation':'xavier',
        'loss_function':'cross_entropy'
        
    }

    # Initialize a new wandb run
    wandb.init(project='test_100', entity='moni6264',config=config_defaults)
    
    
    # Config is a variable that holds and saves hyperparameters and inputs
    config = wandb.config
    learning_rate = config.learning_rate
    epochs = config.epochs
    hidden_layer = config.hidden_layer
    activation = config.activation
    optimiser = config.optimiser
    input_layer = config.input_layer
    output_layer = config.output_layer
    batch_size = config.batch_size
    weight_decay = config.weight_decay
    loss_function = config.loss_function
    initialisation = config.initilisation
    # Model training here
    sweep_network    = ANN(input_layer, hidden_layer, output_layer,initialisation,activation,loss_function)
    acc1,acc2,train_loss,val_loss  = sweep_network.fit_predict(X_train,y_train,X_val,y_val,epochs,learning_rate,optimiser,batch_size,weight_decay)<jupyter_output><empty_output><jupyter_text>Running sweeps configuration<jupyter_code>import warnings
warnings.filterwarnings("ignore", category=np.VisibleDeprecationWarning)
#wandb.agent(sweep_id, train, count=100)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13790894757587702, 'fraction_numerical': 0.026852581789515176, 'mean_word_length': 2.6070730406966414, 'pattern_counts': {'":': 6, '<': 40, '<?xml version=': 0, '>': 41, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '623289', 'license': 'MIT', 'n_tokens_mistral': 7258, 'n_tokens_neox': 6848, 'n_words': 1206, 'path': 'ANNclass.ipynb', 'repo_name': 'dipnarayan501/deep_learning', 'size': '20296'}