<jupyter_start><jupyter_text>Distilling Knowledge in Multiple Students  using Generative models<jupyter_code># !nvidia-smi
import numpy as np
import tensorflow as tf
import tensorflow.keras
import tensorflow.keras.backend as K
from tensorflow.keras.datasets import fashion_mnist,mnist,cifar10
from tensorflow.keras.layers import Conv2D,Activation,BatchNormalization,UpSampling2D,Embedding,ZeroPadding2D, Input, Flatten, Dense, Reshape, LeakyReLU, Dropout,MaxPooling2D
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.optimizers import Adam, SGD, RMSprop
from tensorflow.keras import regularizers

from tensorflow.keras.utils import Progbar
from keras.initializers import RandomNormal
import random
from sklearn.model_selection import train_test_split
from tensorflow.keras import utils as np_utils

physical_devices = tf.config.list_physical_devices('GPU')
tf.config.experimental.set_memory_growth(physical_devices[0], True)<jupyter_output><empty_output><jupyter_text>Teacher's Training<jupyter_code>#Loading and splitting the dataset into train, validation and test
nb_classes = 10

(X_Train, y_Train), (X_test, y_test) = mnist.load_data()
X_train, X_val, y_train, y_val = train_test_split(X_Train, y_Train, test_size=0.20)
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_val = np_utils.to_categorical(y_val, nb_classes)

X_train = X_train.reshape(48000, 28, 28, 1)
X_val = X_val.reshape(12000, 28, 28, 1)

X_train = X_train.astype('float32')
X_val = X_val.astype('float32')

# Normalize the values
X_train /= 255
X_val /= 255


#Creating a teacher network
input_shape = (28, 28, 1) # Input shape of each image

teacher = Sequential()
teacher.add(Conv2D(32, kernel_size=(3, 3),
                 activation='relu',
                 input_shape=input_shape))
teacher.add(Conv2D(64, (3, 3), activation='relu'))
teacher.add(MaxPooling2D(pool_size=(2, 2)))

teacher.add(Dropout(0.25)) # For reguralization

teacher.add(Flatten())
teacher.add(Dense(256, activation='relu'))
teacher.add(Dense(256, activation='relu', name="dense_1"))

teacher.add(Dropout(0.5)) # For reguralization

teacher.add(Dense(nb_classes, name = 'dense_2'))
teacher.add(Activation('softmax')) # Note that we add a normal softmax layer to begin with

teacher.compile(loss='categorical_crossentropy',
              optimizer='adadelta',
              metrics=['accuracy'])

print(teacher.summary())
teacher.load_weights("Teacher_MNIST_98.h5")
# Train the teacher model as usual
epochs = 0
batch_size = 256
teacher.fit(X_train, Y_train,
          batch_size=batch_size,
          epochs=epochs,
          verbose=1,
          validation_data=(X_val, Y_val))


#checking teachers test accuracy
X_test = X_test.reshape(10000, 28, 28, 1)

X_test = X_test.astype('float32')
# Normalize the values
X_test /= 255
preds=teacher.predict(X_test)

numbers=[0,1,2,3,4,5,6,7,8,9]
preds2=[]
for p in preds:
    preds2.append(numbers[np.argmax(p)])
preds2=np.array(preds2)

print (y_test)
print (preds2)
print (np.sum(y_test==preds2))<jupyter_output>[7 2 1 ... 4 5 6]
[7 2 1 ... 4 5 6]
9899<jupyter_text>Evaluation<jupyter_code>Y_test = np_utils.to_categorical(y_test, nb_classes)
X_test = X_test.reshape(10000, 28, 28, 1)
teacher.evaluate(X_test,Y_test)<jupyter_output>313/313 [==============================] - 1s 2ms/step - loss: 0.0541 - accuracy: 0.9869<jupyter_text>Dense Vector split<jupyter_code>teacher_WO_Softmax = Model(teacher.input, teacher.get_layer('dense_1').output)
train_dense = teacher_WO_Softmax.predict(X_train)
val_dense = teacher_WO_Softmax.predict(X_val)

# 4 Students case
# ---------------------------------------------
s1Train=train_dense[:,:64]
s2Train=train_dense[:,64:128]
s3Train=train_dense[:,128:192]
s4Train=train_dense[:,192:]

s1Val=val_dense[:,:64]
s2Val=val_dense[:,64:128]
s3Val=val_dense[:,128:192]
s4Val=val_dense[:,192:]<jupyter_output><empty_output><jupyter_text>Student<jupyter_code>def define_student(name):

    student1 = Sequential()
    student1.add(Conv2D(32, kernel_size=(3, 3),activation='relu',input_shape=(28, 28, 1),kernel_initializer='normal', name=name))
    student1.add(Conv2D(32, (3, 3), activation='relu',kernel_initializer='normal'))
    student1.add(MaxPooling2D(pool_size=(2, 2)))

    student1.add(Conv2D(16, kernel_size=(3, 3),activation='relu',kernel_initializer='normal'))
    student1.add(Conv2D(16, (3, 3), activation='relu',kernel_initializer='normal'))
    student1.add(MaxPooling2D(pool_size=(2, 2)))

    student1.add(Dropout(0.25)) # For reguralization
    student1.add(Flatten())
    student1.add(Dense(8, activation='relu'))
    student1.add(Dropout(0.3))
    student1.add(Dense(64,name='req'+name))

    student1.compile('adam',loss='mean_squared_error',metrics=['accuracy'])
    
    return student1<jupyter_output><empty_output><jupyter_text>You can initialize multiple instances here<jupyter_code>s1=define_student("s1")
s2=define_student('s2')
s3=define_student("s3")
s4=define_student('s4')

s1.fit(X_train, s1Train,
          batch_size=batch_size,
          epochs=10,
          verbose=1, validation_data=(X_val, s1Val))
s2.fit(X_train, s2Train,
          batch_size=batch_size,
          epochs=10,verbose=1, validation_data=(X_val, s2Val))
s3.fit(X_train, s3Train,
          batch_size=batch_size,
          epochs=10,verbose=1, validation_data=(X_val, s3Val))
s4.fit(X_train, s4Train,
          batch_size=batch_size,
          epochs=10,verbose=1, validation_data=(X_val, s4Val))<jupyter_output>Epoch 1/10
188/188 [==============================] - 3s 11ms/step - loss: 2.1434 - accuracy: 0.0492 - val_loss: 1.2337 - val_accuracy: 0.1730
Epoch 2/10
188/188 [==============================] - 2s 8ms/step - loss: 1.3028 - accuracy: 0.1711 - val_loss: 0.7762 - val_accuracy: 0.2561
Epoch 3/10
188/188 [==============================] - 2s 10ms/step - loss: 1.0305 - accuracy: 0.2316 - val_loss: 0.6440 - val_accuracy: 0.3099
Epoch 4/10
188/188 [==============================] - 2s 10ms/step - loss: 0.9530 - accuracy: 0.2431 - val_loss: 0.6111 - val_accuracy: 0.3196
Epoch 5/10
188/188 [==============================] - 2s 11ms/step - loss: 0.9215 - accuracy: 0.2480 - val_loss: 0.5882 - val_accuracy: 0.3078
Epoch 6/10
188/188 [==============================] - 2s 10ms/step - loss: 0.8874 - accuracy: 0.2443 - val_loss: 0.5763 - val_accuracy: 0.3107
Epoch 7/10
188/188 [==============================] - 2s 10ms/step - loss: 0.8680 - accuracy: 0.2462 - val_loss: 0.5281 - val_accuracy: 0.3385
[...]<jupyter_text>**4 Students**<jupyter_code>o1=s1.get_layer("reqs1").output
o2=s2.get_layer("reqs2").output
o3=s3.get_layer("reqs3").output
o4=s4.get_layer("reqs4").output
output=tensorflow.keras.layers.concatenate([o1,o2,o3,o4])
print (output.shape)
output=Activation('relu')(output)
output2=Dropout(0.5)(output) # For reguralization
output3=Dense(10,activation="softmax", name="d1")(output2)
mm4=Model([s1.get_layer("s1").input,s2.get_layer("s2").input,s3.get_layer("s3").input,s4.get_layer("s4").input],output3)
my_weights=teacher.get_layer('dense_2').get_weights()
mm4.get_layer('d1').set_weights(my_weights)
for l in mm4.layers[:len(mm4.layers)-1]:
    l.trainable=False

mm4.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

batch_size = 128
mm2_history=mm4.fit([X_train,X_train,X_train,X_train], Y_train,
          batch_size=batch_size,
          epochs=5,
          verbose=1, validation_data=([X_val,X_val,X_val,X_val], Y_val))
loss, acc = mm4.evaluate([X_test,X_test,X_test,X_test], Y_test, verbose=1)
loss, acc 
s1.summary()
mm4.summary()<jupyter_output>Model: "model_6"
__________________________________________________________________________________________________
Layer (type)                    Output Shape         Param #     Connected to                     
==================================================================================================
s1_input (InputLayer)           [(None, 28, 28, 1)]  0                                            
__________________________________________________________________________________________________
s2_input (InputLayer)           [(None, 28, 28, 1)]  0                                            
__________________________________________________________________________________________________
s3_input (InputLayer)           [(None, 28, 28, 1)]  0                                            
__________________________________________________________________________________________________
s4_input (InputLayer)           [(None, 28, 28, 1)]  0                                      [...]
{'fraction_non_alphanumeric': 0.17162444805949337, 'fraction_numerical': 0.06832442481989309, 'mean_word_length': 4.772635814889336, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '555477', 'license': 'MIT', 'n_tokens_mistral': 3397, 'n_tokens_neox': 3010, 'n_words': 515, 'path': 'MNIST/Feed Forward Files/[4] Students Configuration On MNIST Using FF.ipynb', 'repo_name': 'musab-r/TCN', 'size': '8606'}