<jupyter_start><jupyter_text>**HERE!**<jupyter_code># listOfDesiredAugmentation=['HF', 'None', 'RO', 'RO-RC', 'HF-RC', 'RO-TR', 'None-RC', 'None-TR', 'HF-TR']
# listOfDesiredAugmentation=['HF', 'None', "RO"]  # Best results so far
listOfDesiredAugmentation=['HF', 'None', 'RO', 'HF-RC', 'None-RC', 'None-TR', 'HF-TR']

# listOfDesiredModelNames=["facenet","vggface","openface", "deepface","deepID", "arcFace"]    #all 
# listOfDesiredModelNames=["facenet","vggface","openface","deepID", "deepface"]                # best performance 
listOfDesiredModelNames=[ "facenet"]                # try minimal stuff  
%cd /content/drive/MyDrive/TrustworthyRegression/FaceEncodings/Training
%cd .. 
%cd Training
training_embedings, training_labels= readEmbedings(listOfDesiredAugmentation=listOfDesiredAugmentation,
                                 listOfDesiredModelNames=listOfDesiredModelNames)
training_embedings[0].shape[0]
%cd ..
%cd Test
# listOfDesiredAugmentation=['HF', 'None', 'RO', 'RO-RC', 'HF-RC', 'RO-TR', 'None-RC', 'None-TR', 'HF-TR']
listOfDesiredAugmentation=['None',]
test_embedings, test_labels= readEmbedings(listOfDesiredAugmentation=listOfDesiredAugmentation,
                                 listOfDesiredModelNames=listOfDesiredModelNames)
from tensorflow.keras import regularizers
Heinitializer = tf.keras.initializers.HeNormal()
from tensorflow.keras.layers.experimental import preprocessing


from tensorflow.keras.layers.experimental import preprocessing
normalizer = preprocessing.Normalization(axis=-1)
normalizer.adapt(training_embedings)
# print(normalizer.mean.numpy())<jupyter_output><empty_output><jupyter_text>dropout_posibility=0.06L2_lambda=0.0004init_learning_rate=0.001newModel = tf.keras.models.Sequential([                      pre_trained_model,      tf.keras.layers.GlobalAveragePooling2D(),    tf.keras.Input(shape=(training_embedings[0].shape[0],)),    normalizer,    tf.keras.layers.Dense(16, activation='relu', kernel_regularizer=regularizers.l2(L2_lambda),kernel_initializer=Heinitializer),    tf.keras.layers.Dropout(dropout_posibility),    tf.keras.layers.Dense(8, activation='relu', kernel_regularizer=regularizers.l2(L2_lambda),kernel_initializer=Heinitializer),    tf.keras.layers.Dropout(dropout_posibility),    tf.keras.layers.Dense(4, activation='relu', kernel_regularizer=regularizers.l2(L2_lambda),kernel_initializer=Heinitializer),    tf.keras.layers.Dropout(dropout_posibility),    tf.keras.layers.Dense(1,kernel_initializer=Heinitializer),    ]) newModel.summary()stathera kato apo 17%<jupyter_code># dropout_posibility=0.05
# L2_lambda=0.0003

dropout_posibility=0.01
L2_lambda=0.00001

init_learning_rate=0.001
newModel = tf.keras.models.Sequential([                 
    # pre_trained_model, 
    # tf.keras.layers.GlobalAveragePooling2D(),
    tf.keras.Input(shape=(training_embedings[0].shape[0],)),
    normalizer,
    # tf.keras.layers.Dense(16, activation='relu', kernel_regularizer=regularizers.l2(L2_lambda),kernel_initializer=Heinitializer),
    # tf.keras.layers.Dropout(dropout_posibility),
    # tf.keras.layers.Dense(8, activation='relu', kernel_regularizer=regularizers.l2(L2_lambda),kernel_initializer=Heinitializer),
    # tf.keras.layers.Dropout(dropout_posibility),
    # tf.keras.layers.Dense(4, activation='relu', kernel_regularizer=regularizers.l2(L2_lambda),kernel_initializer=Heinitializer),
    # tf.keras.layers.Dropout(dropout_posibility),
    tf.keras.layers.Dense(1,kernel_initializer=Heinitializer),
    ]) 
newModel.summary()
numberEpochsPerIteration=250
numberOfIterations=4
for i in range(1, numberOfIterations+1):
  learning_rate=init_learning_rate/(i*10)
  adamOptimizerWithCustomLearningRate =tf.keras.optimizers.Adam(learning_rate=learning_rate)
  newModel.compile(
    optimizer=adamOptimizerWithCustomLearningRate,
    # loss=tf.keras.losses.MeanAbsoluteError())
  loss= tf.keras.losses.MeanAbsolutePercentageError())


    # loss=tf.keras.losses.MeanSquaredError())
  
  history= newModel.fit(
    training_embedings,
    training_labels,
    validation_data=(test_embedings, test_labels),
  # train_ds,
  # validation_data=val_ds,
  epochs=numberEpochsPerIteration
  )
  plot_loss(history)





# model_list=[facenet, vggface, openface, deepface, deepID,arcFace ]

# listOfDesiredModelNames=["facenet", "openface", "deepID", "arcFace"]                # try minimal stuff  
model_list=[facenet, deepID ]
fine_tune_at=5
for model in model_list:
  model=model.layers[1]
  for layer in model.layers[:fine_tune_at]:
    layer.trainable =  False
  for layer in model.layers[fine_tune_at:]:
    layer.trainable =  True 
# tempImage=x_test[0]
# inputs = Input(shape=(160, 160, 3))
# model1= vggface(inputs)
# model2= deepID(inputs)
# model3= openface(inputs)
# model4= deepface(inputs)
# model5= deepID(inputs)
# model6= arcFace(inputs)
# branches = [model1, model2,model3, model4,model5, model6]
# mixed = Concatenate(axis=1, name='models_Concatenate')(branches)
# pre_classifier= tf.keras.layers.Dense(newModel.input_shape[1])(mixed)
# classifier= newModel(pre_classifier)

# concatenatedPreTrainedModels = Model(inputs, classifier, name='concatenatedPreTrainedModels')

inputs = Input(shape=(160, 160, 3))
models=list()
for model in model_list:
  models.append(model(inputs))
branches = models
mixed = Concatenate(axis=1, name='models_Concatenate')(branches)
# pre_classifier= tf.keras.layers.Dense(newModel.input_shape[1])(mixed)
classifier= newModel(mixed)

concatenatedPreTrainedModels = Model(inputs, classifier, name='concatenatedPreTrainedModels')


concatenatedPreTrainedModels.output.shape
model=concatenatedPreTrainedModels
learning_rate=0.000011
adamOptimizerWithCustomLearningRate =tf.keras.optimizers.Adam(learning_rate=learning_rate)
model.compile(
  optimizer=adamOptimizerWithCustomLearningRate,
  # loss=tf.keras.losses.MeanAbsoluteError())
loss= tf.keras.losses.MeanAbsolutePercentageError())


  # loss=tf.keras.losses.MeanSquaredError())

history= model.fit(
    x_train_all,
    y_train_all,
  validation_data=(x_test, y_test),
epochs=2
)
plot_loss(history)


# model..com
# def getmixedModel():

#   return model;
# mixedModel= getmixedModel()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13114489003880983, 'fraction_numerical': 0.01908150064683053, 'mean_word_length': 6.242388758782202, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '523536', 'license': 'MIT', 'n_tokens_mistral': 2255, 'n_tokens_neox': 2135, 'n_words': 269, 'path': 'RegressionModelArchitectureCYENS_Trustworthyness.ipynb', 'repo_name': 'StylianosHerodotou/InvestigatingBiasInMLModels', 'size': '6184'}