<jupyter_start><jupyter_text>1. Download the data from here2. Code the model to classify data like below image3. Write your own callback function, that has to print the micro F1 score and AUC score after each epoch.4. Save your model at every epoch if your validation accuracy is improved from previous epoch. 5. you have to decay learning based on below conditions         Cond1. If your validation accuracy at that epoch is less than previous epoch accuracy, you have to decrese the               learning rate by 10%.         Cond2. For every 3rd epoch, decay your learning rate by 5%.        6. If you are getting any NaN values(either weigths or loss) while training, you have to terminate your training. 7. You have to stop the training if your validation accuracy is not increased in last 2 epochs.8. Use tensorboard for every model and analyse your gradients. (you need to upload the screenshots for each model for evaluation)9. use cross entropy as loss function10. Try the architecture params as given below.  Model-11. Use tanh as an activation for every layer except output layer.2. use SGD with momentum as optimizer.3. use RandomUniform(0,1) as initilizer.3. Analyze your output and training process. Model-21. Use relu as an activation for every layer except output layer.2. use SGD with momentum as optimizer.3. use RandomUniform(0,1) as initilizer.3. Analyze your output and training process. Model-31. Use relu as an activation for every layer except output layer.2. use SGD with momentum as optimizer.3. use he_uniform() as initilizer.3. Analyze your output and training process. Model-41. Try with any values to get better accuracy/f1 score.<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
#importing required libraries

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import tensorflow as tf
from sklearn.model_selection import train_test_split
from tensorflow.keras.layers import Dense,Input,Activation
from tensorflow.keras.models import Model
import random as rn
from tensorflow.keras.models import Sequential, load_model 
from tensorflow.keras.layers import Dense, Activation 
from sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score, roc_auc_score
from tensorflow.keras.callbacks import Callback, ModelCheckpoint, TensorBoard, LearningRateScheduler, EarlyStopping, ReduceLROnPlateau
import tensorflow.keras.backend as K
from tensorflow.keras.initializers import RandomUniform, HeUniform
from tensorflow.keras.optimizers import SGD, Adam
import datetime<jupyter_output><empty_output><jupyter_text>Loading data<jupyter_code>data=pd.read_csv('/content/drive/MyDrive/AAIC/20_Working with Callbacks/data.csv')       #load the data
data.shape

plt.plot(data)
plt.show()
data<jupyter_output><empty_output><jupyter_text>Separating class lables and data point(X, Y)<jupyter_code>y=data['label'].values
y
X=data.drop('label', axis=1)
X<jupyter_output><empty_output><jupyter_text>Split the data into train test<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(X, y,test_size=0.33, stratify=y)
X_train.shape, X_test.shape, y_train.shape, y_test.shape
X_train.shape[1]
y_train.shape
#https://www.w3schools.com/python/python_inheritance.asp
#roc_callback funtion

class roc_callback(Callback):
    def __init__(self,train_data,test_data):
        self.x_train = train_data[0]
        self.y_train_ = train_data[1]
        self.x_test = test_data[0]
        self.y_test_ = test_data[1]

    def on_epoch_end(self, epoch, logs={}):                             #after the end of epoch values
        y_train_predicted = self.model.predict(self.x_train)              #y_i^'s for train data
        roc_train = roc_auc_score(self.y_train_, y_train_predicted)
        y_test_predicted = self.model.predict(self.x_test)               #y_i^'s for test data'
        roc_test = roc_auc_score(self.y_test_, y_test_predicted)
        print('\rroc-auc_train: %s - roc-auc_val: %s' % (str(round(roc_train,4)),str(round(roc_test,4))),end=100*' '+'\n')
        return

roc = roc_callback(train_data=(X_train, y_train),
test_data=(X_test, y_test))
def change_in_learning_rate(epoch):
    '''This function returns the reduced learning rate'''  
    initial_learning_rate=0.1
    changed_learning_rate=initial_learning_rate*(1-0.1)**epoch
    return changed_learning_rate
change_in_learning_rate(5)
change_in_learning_rate(50)
#https://www.tensorflow.org/api_docs/python/tf/keras/layers/Dense
#https://www.tensorflow.org/api_docs/python/tf/keras/initializers/RandomUniform

def create_model():
    '''This function returns the created model where function calll happned'''
    return Sequential([
                     Dense(2, input_shape=(2,), kernel_initializer=RandomUniform(minval=-0.05, maxval=0.05, seed=None), name='input_layer'),
                     Dense(512, activation='tanh', kernel_initializer=RandomUniform(minval=-0.05, maxval=0.05, seed=None), name='layer1'),
                     Dense(256, activation='tanh', kernel_initializer=RandomUniform(minval=-0.05, maxval=0.05, seed=None), name='layer2'),
                     Dense(128, activation='tanh', kernel_initializer=RandomUniform(minval=-0.05, maxval=0.05, seed=None), name='layer3'),
                     Dense(256, activation='tanh', kernel_initializer=RandomUniform(minval=-0.05, maxval=0.05, seed=None), name='layer4'),
                     Dense(512, activation='tanh', kernel_initializer=RandomUniform(minval=-0.05, maxval=0.05, seed=None), name='layer5'),
                     Dense(1, activation='sigmoid', name='output_layer'),])
#stop the process if any NaN values occurs

class terminate_NaN(Callback):
    def epoch_end(self, epoch, logs={}):
        loss=logs.get('loss')
        if loss is not None:
            if np.isnan(loss) or np.isinf(loss):
            print("Invalid loss and Process terminate at epoch {}".format(epoch))
            self.model.stop_training=True
test_data=(X_test, y_test)
train_data=(X_train, y_train)
test_data
train_data
#import required librarires
from sklearn.metrics import roc_auc_score

#this class is used to return the f1 score, precision and recall
class metrics(Callback):
    def on_train_begin(self, logs={}):
        self.value_f1_score = []                    #to store f1 score
        self.value_recalls = []                     #to store recall 
        self.value_precisions = []                  #to store precision
    def on_epoch_end(self, epoch, logs={}):
        val_predict = (np.asarray(self.model.predict(X_test))).round()     #to get in numpy array 
        val_targ = y_test
        val_f1 = f1_score(val_targ, val_predict)                           #to get f1 score
        val_recall = recall_score(val_targ, val_predict)                   #to get recall
        val_precision = precision_score(val_targ, val_predict)             #to get precission
        self.value_f1_score.append(val_f1)
        self.value_recalls.append(val_recall)
        self.value_precisions.append(val_precision)
        print ( "- f1 score %f - precision %f - recall %f" %(val_f1, val_precision, val_recall))
        return

metrics_f1_score=metrics()
metrics_f1_score<jupyter_output><empty_output><jupyter_text>**Model 1**<jupyter_code>#https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/SGD
#https://www.tensorflow.org/api_docs/python/tf/keras/Model
#https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/ReduceLROnPlateau
#https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/ModelCheckpoint
#https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/EarlyStopping

#Model1

model_no1=create_model()
optimizer=SGD(learning_rate=0.01, momentum=0.9, nesterov=False, name="SGD")
model_no1.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])

# log_directory="logs/fit/"+datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
# tensorboard_callback=TensorBoard(log_dir=log_directory, histogram_freq=1, write_graph=True)

log_dir="logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir,histogram_freq=1, write_graph=True,write_grads=True)

terminate_nan=terminate_NaN()
file_path="model_save/weights-{epoch:02d}-{val_accuracy:.4f}.hdf5"

check_point=ModelCheckpoint(filepath=file_path, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')
early_stop=EarlyStopping(monitor='val_accuracy', patience=2, verbose=1, mode='max')
reduce_accuracy=ReduceLROnPlateau(monitor='val_accuracy', factor=0.9, patience=1, verbose=1, mode='auto')
reduce_lr_for_3epoch=ReduceLROnPlateau(factor=0.9, patience=3, verbose=1)

callbacks_list=[check_point, tensorboard_callback, early_stop, reduce_accuracy, reduce_lr_for_3epoch, terminate_nan, metrics_f1_score, roc]

model_no1.fit(x=X_train,
              y=y_train,
              epochs=15,
              validation_data=(X_test, y_test),
              callbacks=[callbacks_list])
model_no1.summary()
%reload_ext tensorboard
!rm -rf ./logs/
%tensorboard --logdir='logs/fit'<jupyter_output><empty_output><jupyter_text>Model 2<jupyter_code>def create_model():
    '''This function returns the created model where function calll happned'''
    return Sequential([
                     Dense(2, input_shape=(2,), kernel_initializer=RandomUniform(minval=-0.05, maxval=0.05, seed=None), name='input_layer'),
                     Dense(512, activation='relu', kernel_initializer=RandomUniform(minval=-0.05, maxval=0.05, seed=None), name='layer1'),
                     Dense(256, activation='relu', kernel_initializer=RandomUniform(minval=-0.05, maxval=0.05, seed=None), name='layer2'),
                     Dense(128, activation='relu', kernel_initializer=RandomUniform(minval=-0.05, maxval=0.05, seed=None), name='layer3'),
                     Dense(256, activation='relu', kernel_initializer=RandomUniform(minval=-0.05, maxval=0.05, seed=None), name='layer4'),
                     Dense(512, activation='relu', kernel_initializer=RandomUniform(minval=-0.05, maxval=0.05, seed=None), name='layer5'),
                     Dense(1, activation='sigmoid', name='output_layer'),])
#https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/SGD
#https://www.tensorflow.org/api_docs/python/tf/keras/Model
#https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/ReduceLROnPlateau
#https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/ModelCheckpoint
#https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/EarlyStopping


#Model2

model_no2=create_model()
optimizer=SGD(learning_rate=0.01, momentum=0.9, nesterov=False, name="SGD")
model_no2.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])

# log_directory="logs/fit/"+datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
# tensorboard_callback=TensorBoard(log_dir=log_directory, histogram_freq=1, write_graph=True)

log_dir="logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir,histogram_freq=1, write_graph=True,write_grads=True)

terminate_nan=terminate_NaN()
file_path="model_save/weights-{epoch:02d}-{val_accuracy:.4f}.hdf5"

check_point=ModelCheckpoint(filepath=file_path, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')
early_stop=EarlyStopping(monitor='val_accuracy', patience=2, verbose=1, mode='max')
reduce_accuracy=ReduceLROnPlateau(monitor='val_accuracy', factor=0.9, patience=1, verbose=1, mode='auto')
reduce_lr_for_3epoch=ReduceLROnPlateau(factor=0.9, patience=3, verbose=1)

callbacks_list=[check_point, tensorboard_callback, early_stop, reduce_accuracy, reduce_lr_for_3epoch, terminate_nan, metrics_f1_score, roc]

model_no2.fit(x=X_train,
              y=y_train,
              epochs=15,
              validation_data=(X_test, y_test),
              callbacks=[callbacks_list])
model_no2.summary()

%reload_ext tensorboard

%tensorboard --logdir='/content/logs/fit/20210921-100537'<jupyter_output><empty_output><jupyter_text>Model 3<jupyter_code>def create_model():
  '''This function returns the created model where function calll happned'''
  return Sequential([
                     Dense(2, input_shape=(2,), kernel_initializer=HeUniform(), name='input_layer'),
                     Dense(512, activation='relu', kernel_initializer=HeUniform(), name='layer1'),
                     Dense(256, activation='relu', kernel_initializer=HeUniform(), name='layer2'),
                     Dense(128, activation='relu', kernel_initializer=HeUniform(), name='layer3'),
                     Dense(256, activation='relu', kernel_initializer=HeUniform(), name='layer4'),
                     Dense(512, activation='relu', kernel_initializer=HeUniform(), name='layer5'),
                     Dense(1, activation='sigmoid', name='output_layer'),
  ])
#https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/SGD
#https://www.tensorflow.org/api_docs/python/tf/keras/Model
#https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/ReduceLROnPlateau
#https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/ModelCheckpoint
#https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/EarlyStopping

#Model2

model_no3=create_model()
optimizer=SGD(learning_rate=0.01, momentum=0.9, nesterov=False, name="SGD")
model_no3.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])

# log_directory="logs/fit/"+datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
# tensorboard_callback=TensorBoard(log_dir=log_directory, histogram_freq=1, write_graph=True)

log_dir="logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir,histogram_freq=1, write_graph=True,write_grads=True)

terminate_nan=terminate_NaN()
file_path="model_save/weights-{epoch:02d}-{val_accuracy:.4f}.hdf5"

check_point=ModelCheckpoint(filepath=file_path, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')
early_stop=EarlyStopping(monitor='val_accuracy', patience=2, verbose=1, mode='max')
reduce_accuracy=ReduceLROnPlateau(monitor='val_accuracy', factor=0.9, patience=1, verbose=1, mode='auto')
reduce_lr_for_3epoch=ReduceLROnPlateau(factor=0.9, patience=3, verbose=1)

callbacks_list=[check_point, tensorboard_callback, early_stop, reduce_accuracy, reduce_lr_for_3epoch, terminate_nan, metrics_f1_score, roc]

model_no3.fit(x=X_train,
              y=y_train,
              epochs=15,
              validation_data=(X_test, y_test),
              callbacks=[callbacks_list])
model_no3.summary()
%reload_ext tensorboard

%tensorboard --logdir='/content/logs/fit/20210921-101723'<jupyter_output><empty_output><jupyter_text>Model 4<jupyter_code>def create_model():
  '''This function returns the created model where function calll happned'''
  return Sequential([
                     Dense(2, input_shape=(2,), kernel_initializer=HeUniform(), name='input_layer'),
                     Dense(512, activation='softmax', kernel_initializer=HeUniform(), name='layer1'),
                     Dense(256, activation='softmax', kernel_initializer=HeUniform(), name='layer2'),
                     Dense(128, activation='softmax', kernel_initializer=HeUniform(), name='layer3'),
                     Dense(256, activation='softmax', kernel_initializer=HeUniform(), name='layer4'),
                     Dense(512, activation='softmax', kernel_initializer=HeUniform(), name='layer5'),
                     Dense(1, activation='softmax', name='output_layer'),
  ])
#https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/SGD
#https://www.tensorflow.org/api_docs/python/tf/keras/Model
#https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/ReduceLROnPlateau
#https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/ModelCheckpoint
#https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/EarlyStopping



#Model4

model_no4=create_model()
optimizer=SGD(learning_rate=0.01, momentum=0.9, nesterov=False, name="SGD")
model_no4.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])

# log_directory="logs/fit/"+datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
# tensorboard_callback=TensorBoard(log_dir=log_directory, histogram_freq=1, write_graph=True)

log_dir="logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir,histogram_freq=1, write_graph=True,write_grads=True)

terminate_nan=terminate_NaN()
file_path="model_save/weights-{epoch:02d}-{val_accuracy:.4f}.hdf5"

check_point=ModelCheckpoint(filepath=file_path, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')
early_stop=EarlyStopping(monitor='val_accuracy', patience=2, verbose=1, mode='max')
reduce_accuracy=ReduceLROnPlateau(monitor='val_accuracy', factor=0.9, patience=1, verbose=1, mode='auto')
reduce_lr_for_3epoch=ReduceLROnPlateau(factor=0.9, patience=3, verbose=1)

callbacks_list=[check_point, tensorboard_callback, early_stop, reduce_accuracy, reduce_lr_for_3epoch, terminate_nan, metrics_f1_score, roc]

model_no4.fit(x=X_train,
              y=y_train,
              epochs=15,
              validation_data=(X_test, y_test),
              callbacks=[callbacks_list])
model_no4.summary()
%reload_ext tensorboard

%tensorboard --logdir='/content/logs/fit/20210921-102631'<jupyter_output><empty_output><jupyter_text>Model 5<jupyter_code>def create_model():
  '''This function returns the created model where function calll happned'''
  return Sequential([
                     Dense(2, input_shape=(2,), kernel_initializer=HeUniform(), name='input_layer'),
                     Dense(512, activation='relu', kernel_initializer=HeUniform(), name='layer1'),
                     Dense(256, activation='relu', kernel_initializer=HeUniform(), name='layer2'),
                     Dense(128, activation='relu', kernel_initializer=HeUniform(), name='layer3'),
                     Dense(256, activation='relu', kernel_initializer=HeUniform(), name='layer4'),
                     Dense(512, activation='relu', kernel_initializer=HeUniform(), name='layer5'),
                     Dense(1, activation='softmax', name='output_layer'),
  ])
#https://keras.io/api/optimizers/
#https://www.tensorflow.org/api_docs/python/tf/keras/optimizers/SGD
#https://www.tensorflow.org/api_docs/python/tf/keras/Model
#https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/ReduceLROnPlateau
#https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/ModelCheckpoint
#https://www.tensorflow.org/api_docs/python/tf/keras/callbacks/EarlyStopping
#Model5

model_no5=create_model()
# optimizer=SGD(learning_rate=0.01, momentum=0.9, nesterov=False, name="SGD")
optimizer = tf.keras.optimizers.Adam(learning_rate=0.01)
model_no5.compile(optimizer=optimizer, loss='binary_crossentropy', metrics=['accuracy'])

# log_directory="logs/fit/"+datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
# tensorboard_callback=TensorBoard(log_dir=log_directory, histogram_freq=1, write_graph=True)

log_dir="logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir,histogram_freq=1, write_graph=True,write_grads=True)

terminate_nan=terminate_NaN()
file_path="model_save/weights-{epoch:02d}-{val_accuracy:.4f}.hdf5"

check_point=ModelCheckpoint(filepath=file_path, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')
early_stop=EarlyStopping(monitor='val_accuracy', patience=2, verbose=1, mode='max')
reduce_accuracy=ReduceLROnPlateau(monitor='val_accuracy', factor=0.9, patience=1, verbose=1, mode='auto')
reduce_lr_for_3epoch=ReduceLROnPlateau(factor=0.9, patience=3, verbose=1)

callbacks_list=[check_point, tensorboard_callback, early_stop, reduce_accuracy, reduce_lr_for_3epoch, terminate_nan, metrics_f1_score, roc]

model_no5.fit(x=X_train,
              y=y_train,
              epochs=15,
              validation_data=(X_test, y_test),
              callbacks=[callbacks_list])
model_no5.summary()
%reload_ext tensorboard

%tensorboard --logdir='/content/logs/fit/20210921-103411'            #visalize throgh tensorboard<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12945544554455446, 'fraction_numerical': 0.024257425742574258, 'mean_word_length': 4.937977660199882, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 29, 'lorem ipsum': 0, 'www.': 28, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '580773', 'license': 'Unlicense', 'n_tokens_mistral': 6974, 'n_tokens_neox': 6696, 'n_words': 1116, 'path': 'Call_Backs_Assignment.ipynb', 'repo_name': 'charanhu/Call-Backs-using-Deep-Learning', 'size': '20200'}