<jupyter_start><jupyter_text>Distilling Knowledge in Multiple Students Using Generative Models<jupyter_code># %tensorflow_version 1.x
# !pip install --upgrade opencv-python==3.4.2.17
import numpy as np
import tensorflow as tf
import tensorflow.keras
import tensorflow.keras.backend as K
# import os
from tensorflow.keras.datasets import fashion_mnist,mnist,cifar10
# import keras.backend as K
from tensorflow.keras.layers import Conv2D,Activation,BatchNormalization,UpSampling2D,Embedding,ZeroPadding2D, Input, Flatten, Dense, Reshape, LeakyReLU, Dropout,MaxPooling2D
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.optimizers import Adam, SGD, RMSprop
from tensorflow.keras import regularizers

from tensorflow.keras.utils import Progbar
from keras.initializers import RandomNormal
import random
from sklearn.model_selection import train_test_split
# from keras.utils import np_utils
from tensorflow.keras import utils as np_utils


#Loading and splitting the dataset into train, validation and test
nb_classes = 10

(X_Train, y_Train), (X_test, y_test) = fashion_mnist.load_data()
X_train, X_val, y_train, y_val = train_test_split(X_Train, y_Train, test_size=0.20)
# convert y_train and y_test to categorical binary values 
Y_train = np_utils.to_categorical(y_train, nb_classes)
Y_val = np_utils.to_categorical(y_val, nb_classes)

X_train = X_train.reshape(48000, 28, 28, 1)
X_val = X_val.reshape(12000, 28, 28, 1)

X_train = X_train.astype('float32')
X_val = X_val.astype('float32')

# Normalize the values
X_train /= 255
X_val /= 255


#Creating a teacher network
input_shape = (28, 28, 1) # Input shape of each image

teacher = Sequential()
teacher.add(Conv2D(32, kernel_size=(3, 3),
                 activation='relu',
                 input_shape=input_shape))
teacher.add(Conv2D(64, (3, 3), activation='relu'))
teacher.add(MaxPooling2D(pool_size=(2, 2)))

teacher.add(Dropout(0.25)) # For reguralization

teacher.add(Flatten())
teacher.add(Dense(256, activation='relu'))
teacher.add(Dense(256, activation='relu', name="dense_1"))

teacher.add(Dropout(0.5)) # For reguralization

teacher.add(Dense(nb_classes, name = 'dense_2'))
teacher.add(Activation('softmax')) # Note that we add a normal softmax layer to begin with

teacher.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

print(teacher.summary())

# Train the teacher model as usual
# epochs = 0
# batch_size = 256
# teacher.fit(X_train, Y_train,
#           batch_size=batch_size,
#           epochs=epochs,
#           verbose=1,
#           validation_data=(X_val, Y_val))
teacher.load_weights("Teacher_FMNIST_92.h5")
Y_test = np_utils.to_categorical(y_test, nb_classes)
X_test = X_test.reshape(10000, 28, 28, 1)
X_test = X_test.astype('float32')
X_test /= 255


teacher.evaluate(X_test,Y_test)
teacher_WO_Softmax = Model(teacher.input, teacher.get_layer('dense_1').output)
train_dense = teacher_WO_Softmax.predict(X_train)
val_dense = teacher_WO_Softmax.predict(X_val)


# 4 Students case
# ---------------------------------------------
s1Train=train_dense[:,:64]
s2Train=train_dense[:,64:128]
s3Train=train_dense[:,128:192]
s4Train=train_dense[:,192:]<jupyter_output><empty_output><jupyter_text>GANs' Training<jupyter_code># import np.random import random
BATCH_SIZE=32
def smooth_real_labels(y):
    return y - 0.3+(np.random.random(y.shape)*0.5)
def smooth_fake_labels(y):
    return y + (0.3 * np.random.random(y.shape))
def build_gan(gen,disc): 
    disc.trainable = False
    input= Input(shape=input_shape)
    output = gen(input)
    output2= disc(output)
    gan=Model(input,output2)

    gan.compile(Adam(lr=0.0002),loss=['binary_crossentropy','mse'],metrics=['accuracy'])

    return gan
def build_sdiscriminator():
    
    input2 = Input(shape=(64,),name='input')
    inp=Dense(64,use_bias=False)(input2)

    leaky_relu = LeakyReLU(alpha=0.2)(inp)
    
    conv3 = Dense(128)(leaky_relu)
    b_n = BatchNormalization()(conv3)
    leaky_relu = LeakyReLU(alpha=0.2)(b_n)
    conv4 = Dense(256)(leaky_relu)
    b_n = BatchNormalization()(conv4)
    leaky_relu = LeakyReLU(alpha=0.2)(b_n)
    conv4 = Dense(512)(leaky_relu)
    b_n = BatchNormalization()(conv4)
    leaky_relu = LeakyReLU(alpha=0.2)(b_n)
    # conv4 = Dense(1024)(leaky_relu)
    # b_n = BatchNormalization()(conv4)
    # leaky_relu = LeakyReLU(alpha=0.2)(b_n)

    dense = Dense(1,activation='sigmoid',name='dense')(leaky_relu)
    output2=Dense(64)(leaky_relu)
    disc = Model(input2,[dense,output2])          
    disc.compile(optd,loss=['binary_crossentropy','mse'],metrics=['accuracy'])

    return disc
optd = Adam(lr=0.0002)
opt = Adam(lr=0.0002)
def build_sgenerator(name):

    student1 = Sequential()
    student1.add(Conv2D(32, kernel_size=(3, 3),activation='relu',input_shape=(28, 28, 1),kernel_initializer='normal', name=name))
    student1.add(Conv2D(32, (3, 3), activation='relu',kernel_initializer='normal'))
    student1.add(MaxPooling2D(pool_size=(2, 2)))

    student1.add(Conv2D(16, kernel_size=(3, 3),activation='relu',kernel_initializer='normal'))
    student1.add(Conv2D(16, (3, 3), activation='relu',kernel_initializer='normal'))
    student1.add(MaxPooling2D(pool_size=(2, 2)))

    student1.add(Dropout(0.25)) # For reguralization
    student1.add(Flatten())
    student1.add(Dense(16, activation='relu'))
    student1.add(Dropout(0.3))
    student1.add(Dense(64,name='req'+name))

    student1.compile(opt,loss='mean_squared_error',metrics=['accuracy'])

    return student1


def training(generator,discriminator,gan,features,epo=20):
    BATCH_SIZE = 128
    discriminator.trainable = True
    total_size = X_train.shape[0]
    indices = np.arange(0,total_size ,BATCH_SIZE)
    all_disc_loss = []
    all_gen_loss = []
    all_class_loss=[]
    if total_size % BATCH_SIZE:
        indices = indices[:-1]
    for e in range(epo):
        
        progress_bar = Progbar(target=len(indices))
        np.random.shuffle(indices)
        epoch_gen_loss = []
        epoch_disc_loss = []
        epoch_class_loss= []
        for i,index in enumerate(indices):
        
            inputs=X_train[index:index+BATCH_SIZE]
            strain = features[index:index+BATCH_SIZE]
            y_real = np.ones((BATCH_SIZE,1))
            y_fake = np.zeros((BATCH_SIZE,1))

            #Generator Training
            fake_images = generator.predict_on_batch(inputs)

            #Disrciminator Training
            disc_real_loss1,_,disc_real_loss2,_,_= discriminator.train_on_batch(strain,[y_real,strain])
            disc_fake_loss1,_,disc_fake_loss2,_,_= discriminator.train_on_batch(fake_images,[y_fake,strain])

            #Gans Training
            discriminator.trainable = False
            gan_loss,_,gan_loss2,_,_ = gan.train_on_batch(inputs, [y_real,strain])

            discriminator.trainable = True

            disc_loss = (disc_fake_loss1 + disc_real_loss1)/2
            epoch_disc_loss.append(disc_loss)
            progress_bar.update(i+1)

            epoch_gen_loss.append((gan_loss))

        avg_epoch_disc_loss = np.array(epoch_disc_loss).mean()
        avg_epoch_gen_loss = np.array(epoch_gen_loss).mean()
        all_disc_loss.append(avg_epoch_disc_loss)
        all_gen_loss.append(avg_epoch_gen_loss)
        print("Epoch: %d | Discriminator Loss: %f | Generator Loss: %f | " % (e+1,avg_epoch_disc_loss,avg_epoch_gen_loss))

    return generator
discriminator1 = build_sdiscriminator()
discriminator2 = build_sdiscriminator()
discriminator3 = build_sdiscriminator()
discriminator4 = build_sdiscriminator()
s1=build_sgenerator("s1")
s2=build_sgenerator('s2')
s3=build_sgenerator("s3")
s4=build_sgenerator('s4')
gan1 = build_gan(s1,discriminator1)
gan2 = build_gan(s2,discriminator2)
gan3 = build_gan(s3,discriminator3)
gan4 = build_gan(s4,discriminator4)
s1 = training(s1,discriminator1,gan1,s1Train,epo=55)
s2 = training(s2,discriminator2,gan2,s2Train,epo=55)
s3 = training(s3,discriminator3,gan3,s3Train,epo=60)
s4 = training(s4,discriminator4,gan4,s4Train,epo=58)<jupyter_output>375/375 [==============================] - 14s 34ms/step
Epoch: 1 | Discriminator Loss: 0.656007 | Generator Loss: 1.407748 | 
375/375 [==============================] - 13s 34ms/step
Epoch: 2 | Discriminator Loss: 0.280069 | Generator Loss: 1.007795 | 
375/375 [==============================] - 12s 33ms/step
Epoch: 3 | Discriminator Loss: 0.209719 | Generator Loss: 0.849852 | 
375/375 [==============================] - 13s 34ms/step
Epoch: 4 | Discriminator Loss: 0.178830 | Generator Loss: 0.779656 | 
375/375 [==============================] - 13s 34ms/step
Epoch: 5 | Discriminator Loss: 0.158482 | Generator Loss: 0.707628 | 
375/375 [==============================] - 12s 33ms/step
Epoch: 6 | Discriminator Loss: 0.145496 | Generator Loss: 0.660817 | 
375/375 [==============================] - 13s 34ms/step
Epoch: 7 | Discriminator Loss: 0.137045 | Generator Loss: 0.631385 | 
375/375 [==============================] - 13s 35ms/step
Epoch: 8 | Discriminator Loss: 0.131711 | Generator Lo[...]<jupyter_text>**4 Student**<jupyter_code>o1=s1.get_layer("reqs1").output
o2=s2.get_layer("reqs2").output
o3=s3.get_layer("reqs3").output
o4=s4.get_layer("reqs4").output
output=tensorflow.keras.layers.concatenate([o1,o2,o3,o4])
print (output.shape)
output=Activation('relu')(output)
output2=Dropout(0.5)(output) # For reguralization
output3=Dense(10,activation="softmax", name="d1")(output2)
mm4=Model([s1.get_layer("s1").input,s2.get_layer("s2").input,s3.get_layer("s3").input,s4.get_layer("s4").input],output3)
my_weights=teacher.get_layer('dense_2').get_weights()
mm4.get_layer('d1').set_weights(my_weights)
for l in mm4.layers[:len(mm4.layers)-1]:
    l.trainable=False

mm4.compile(loss='categorical_crossentropy',
              optimizer='adam',
              metrics=['accuracy'])

batch_size = 128
mm2_history=mm4.fit([X_train,X_train,X_train,X_train], Y_train,
          batch_size=batch_size,
          epochs=30,
          verbose=1, validation_data=([X_val,X_val,X_val,X_val], Y_val))
loss, acc = mm4.evaluate([X_test,X_test,X_test,X_test], Y_test, verbose=1)
loss, acc<jupyter_output>313/313 [==============================] - 4s 12ms/step - loss: 0.3835 - accuracy: 0.8626
{'fraction_non_alphanumeric': 0.15978928884986832, 'fraction_numerical': 0.05667739732709004, 'mean_word_length': 4.7692740574001125, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '544145', 'license': 'MIT', 'n_tokens_mistral': 4137, 'n_tokens_neox': 3757, 'n_words': 600, 'path': 'FashionMNIST/Generative Adversarial Networks Files/[4] Student Configurations on FashionMNIST using GANs.ipynb', 'repo_name': 'musab-r/TCN', 'size': '10251'}