<jupyter_start><jupyter_text>Ordering by metrics and retraining phase Dataset: GTSRB  Experiment configuration 1- Date: 27/10/2021  Configuration 1	1. Incremental guided retraining starting from scratch using the new adversarial inputs and original training set<jupyter_code>pip install --user tensorflow==2.5
import argparse

import numpy as np
import tensorflow as tf
import keras.backend as K

import matplotlib.pyplot as plt

from keras.utils import np_utils
from keras.models import Sequential
from keras.layers import Dense, Dropout, Activation, Flatten, Conv2D, MaxPooling2D
from keras.regularizers import l2

import glob
import os
import cv2
import keras


import time
import argparse

from tqdm import tqdm

from keras.models import load_model, Model

tf.__version__ == '2.5.0' # Version of tensorflow
cd '../utils'
# utils for project
import utils_guided_retraining as utils
cd '../notebooks/'
dataset = "gtsrb"
x_train,y_train = utils.get_data(dataset,"Train")
x_val,y_val = utils.get_data(dataset,"Val")
x_test,y_test = utils.get_data(dataset,"Test")
x_train_and_adversary,y_train_and_adversary = utils.get_data(dataset,"Train_and_adversary")
x_adversary_training = x_train_and_adversary[len(x_train):]
print(len(x_adversary_training))
y_adversary_training = y_train_and_adversary[len(y_train):]

print(len(y_adversary_training))
# Obtaining adversarial examples for testing 
x_test_and_adversary,y_test_and_adversary = utils.get_adversarial_data(dataset,'Test_fgsm') 
x_adversary_test_fgsm = x_test_and_adversary[len(x_test):]
print(len(x_adversary_test_fgsm))
y_adversary_test_fgsm = y_test_and_adversary[len(y_test):]

print(len(y_adversary_test_fgsm))<jupyter_output>3923
3923<jupyter_text>----<jupyter_code># Original model 
model_dir = "C:/Users/fjdur/Desktop/upc/project_notebooks/github_project/DL_notebooks/models/tf_model_25-06"

model_original = utils.My_model(dataset,True, model_dir)<jupyter_output>Model loaded correctly<jupyter_text>Loading LSA and DSA values<jupyter_code>save_dir_lsa = "C:/Users/fjdur/Desktop/upc/project_notebooks/github_project/DL_notebooks/SA_values/lsa_values_1.npy"
 
save_dir_dsa = "C:/Users/fjdur/Desktop/upc/project_notebooks/github_project/DL_notebooks/SA_values/dsa_values_1.npy"

target_lsa = np.load(save_dir_lsa)
target_dsa = np.load(save_dir_dsa)
lsa_values = target_lsa
dsa_values = target_dsa

# Obtaining top n images by LSA values
top_images_by_lsa = utils.get_x_of_indexes(list(np.flip(np.argsort(lsa_values))),x_train_and_adversary)
top_labels_by_lsa = utils.get_x_of_indexes(list(np.flip(np.argsort(lsa_values))),y_train_and_adversary)


top_images_by_dsa = utils.get_x_of_indexes(list(np.flip(np.argsort(dsa_values))),x_train_and_adversary)
top_labels_by_dsa = utils.get_x_of_indexes(list(np.flip(np.argsort(dsa_values))),y_train_and_adversary)

len(top_images_by_lsa)//20
len(x_train)
m = 1800
n = 0
image_sets_lsa = []
label_sets_lsa = []


for i in range(len(top_images_by_lsa)//m):
    print(i,":")
    if (i+1 >= len(top_images_by_lsa)//m):
        print("Last")
        print(0," -> ",n+m+(len(top_images_by_lsa)%m))
        top_images_by_lsa_n = np.array(top_images_by_lsa[:n+m+(len(top_images_by_lsa)%m)])
        top_labels_by_lsa_n = np.array(top_labels_by_lsa[:n+m+(len(top_images_by_lsa)%m)])
    else:
        print(0," -> ",m+n)
        top_images_by_lsa_n = np.array(top_images_by_lsa[:n+m])
        top_labels_by_lsa_n = np.array(top_labels_by_lsa[:n+m])
    image_sets_lsa.append(top_images_by_lsa_n)
    label_sets_lsa.append(top_labels_by_lsa_n)
    print(len(top_images_by_lsa_n))
    n += m<jupyter_output>0 :
0  ->  1800
1800
1 :
0  ->  3600
3600
2 :
0  ->  5400
5400
3 :
0  ->  7200
7200
4 :
0  ->  9000
9000
5 :
0  ->  10800
10800
6 :
0  ->  12600
12600
7 :
0  ->  14400
14400
8 :
0  ->  16200
16200
9 :
0  ->  18000
18000
10 :
0  ->  19800
19800
11 :
0  ->  21600
21600
12 :
0  ->  23400
23400
13 :
0  ->  25200
25200
14 :
0  ->  27000
27000
15 :
0  ->  28800
28800
16 :
0  ->  30600
30600
17 :
0  ->  32400
32400
18 :
0  ->  34200
34200
19 :
Last
0  ->  36366
36366<jupyter_text>Training guided by LSA<jupyter_code>print(model_dir)

models_lsa = []
for i in range(len(label_sets_lsa)):
    print(i,":")
    model = utils.My_model(dataset,False,None)
    model.compile_model()
    models_lsa.append(model)

n = 0
models_lsa[n].fit_model(image_sets_lsa[n],label_sets_lsa[n],x_val,y_val)
n=n+1
n=1
models_lsa[n].fit_model(image_sets_lsa[n],label_sets_lsa[n],x_val,y_val)
n = n+1
print(n)
models_lsa[n].fit_model(image_sets_lsa[n],label_sets_lsa[n],x_val,y_val)
n = n+1
print(n)
models_lsa[n].fit_model(image_sets_lsa[n],label_sets_lsa[n],x_val,y_val)
n = n+1
print(n) #4
models_lsa[n].fit_model(image_sets_lsa[n],label_sets_lsa[n],x_val,y_val)
n = n+1
n=5
print(n)
models_lsa[n].fit_model(image_sets_lsa[n],label_sets_lsa[n],x_val,y_val)
n = n+1
print(n)
models_lsa[n].fit_model(image_sets_lsa[n],label_sets_lsa[n],x_val,y_val)
n = n+1
print(n)
models_lsa[n].fit_model(image_sets_lsa[n],label_sets_lsa[n],x_val,y_val)
n = n+1
print(n)
models_lsa[n].fit_model(image_sets_lsa[n],label_sets_lsa[n],x_val,y_val)
n = n+1
print(n)#9
models_lsa[n].fit_model(image_sets_lsa[n],label_sets_lsa[n],x_val,y_val)
n = n+1
## print(n)
models_lsa[n].fit_model(image_sets_lsa[n],label_sets_lsa[n],x_val,y_val)
n = n+1
print(n)
models_lsa[n].fit_model(image_sets_lsa[n],label_sets_lsa[n],x_val,y_val)
n = n+1
print(n)
models_lsa[n].fit_model(image_sets_lsa[n],label_sets_lsa[n],x_val,y_val)
n = n+1
print(n)
models_lsa[n].fit_model(image_sets_lsa[n],label_sets_lsa[n],x_val,y_val)
n = n+1
print(n)#14
models_lsa[n].fit_model(image_sets_lsa[n],label_sets_lsa[n],x_val,y_val)
n = n+1
n=15
print(n)
models_lsa[n].fit_model(image_sets_lsa[n],label_sets_lsa[n],x_val,y_val)
n = n+1
print(n)
models_lsa[n].fit_model(image_sets_lsa[n],label_sets_lsa[n],x_val,y_val)
n = n+1
print(n)
models_lsa[n].fit_model(image_sets_lsa[n],label_sets_lsa[n],x_val,y_val)
n = n+1
print(n)
models_lsa[n].fit_model(image_sets_lsa[n],label_sets_lsa[n],x_val,y_val)
n = n+1
print(n)#19
models_lsa[n].fit_model(image_sets_lsa[n],label_sets_lsa[n],x_val,y_val)
n = n+1
loading = True

models_lsa = []

if loading:
    for i in range(20):
        model_lsa_dir = "D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_lsa_e2_"+str(i)
        print(model_lsa_dir)
        model =utils.My_model('gtsrb',True,model_lsa_dir)
        model.model.compile(loss= 'categorical_crossentropy', optimizer = 'rmsprop', metrics = ['accuracy',tf.keras.metrics.Precision(), tf.keras.metrics.Recall()])
        models_lsa.append(model)<jupyter_output>D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_lsa_e2_0
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_lsa_e2_1
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_lsa_e2_2
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_lsa_e2_3
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_lsa_e2_4
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_lsa_e2_5
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_lsa_e2_6
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_lsa_e2_7
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_lsa_e2_8
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_lsa_e2_9
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_lsa_e2_10
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_lsa_e2_11
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_lsa_e2_12
Model loaded correc[...]<jupyter_text>Training guided by DSA<jupyter_code>m = 1800
n = 0
image_sets_dsa = []
label_sets_dsa = []


for i in range(len(top_images_by_dsa)//m):
    print(i,":")
    if (i+1 >= len(top_images_by_dsa)//m):
        print("Last")
        print(0," -> ",n+m+(len(top_images_by_dsa)%m))
        top_images_by_dsa_n = np.array(top_images_by_dsa[:n+m+(len(top_images_by_dsa)%m)])
        top_labels_by_dsa_n = np.array(top_labels_by_dsa[:n+m+(len(top_images_by_dsa)%m)])
    else:
        print(0," -> ",m+n)
        top_images_by_dsa_n = np.array(top_images_by_dsa[:n+m])
        top_labels_by_dsa_n = np.array(top_labels_by_dsa[:n+m])
    image_sets_dsa.append(top_images_by_dsa_n)
    label_sets_dsa.append(top_labels_by_dsa_n)
    print(len(top_images_by_dsa_n))
    n += m


print(model_dir)

models_dsa = []
for i in range(len(label_sets_dsa)):
    print(i,":")
    model = utils.My_model('gtsrb',False,None)
    model.compile_model()
    models_dsa.append(model)

n=0
n=0
print(n)

models_dsa[n].fit_model(image_sets_dsa[n],label_sets_dsa[n],x_val,y_val)
n = n+1
n=1
print(n)

models_dsa[n].fit_model(image_sets_dsa[n],label_sets_dsa[n],x_val,y_val)
n = n+1
n=2
print(n)

models_dsa[n].fit_model(image_sets_dsa[n],label_sets_dsa[n],x_val,y_val)
n = n+1
print(n)

models_dsa[n].fit_model(image_sets_dsa[n],label_sets_dsa[n],x_val,y_val)
n = n+1
print(n)#4

models_dsa[n].fit_model(image_sets_dsa[n],label_sets_dsa[n],x_val,y_val)
n = n+1
n=5
print(n)

models_dsa[n].fit_model(image_sets_dsa[n],label_sets_dsa[n],x_val,y_val)
n = n+1
print(n)

models_dsa[n].fit_model(image_sets_dsa[n],label_sets_dsa[n],x_val,y_val)
n = n+1
print(n)

models_dsa[n].fit_model(image_sets_dsa[n],label_sets_dsa[n],x_val,y_val)
n = n+1
print(n)

models_dsa[n].fit_model(image_sets_dsa[n],label_sets_dsa[n],x_val,y_val)
n = n+1
print(n)#9

models_dsa[n].fit_model(image_sets_dsa[n],label_sets_dsa[n],x_val,y_val)
n = n+1
n=10
print(n)

models_dsa[n].fit_model(image_sets_dsa[n],label_sets_dsa[n],x_val,y_val)
n = n+1
print(n)

models_dsa[n].fit_model(image_sets_dsa[n],label_sets_dsa[n],x_val,y_val)
n = n+1
print(n)

models_dsa[n].fit_model(image_sets_dsa[n],label_sets_dsa[n],x_val,y_val)
n = n+1
print(n)

models_dsa[n].fit_model(image_sets_dsa[n],label_sets_dsa[n],x_val,y_val)
n = n+1
print(n)#14

models_dsa[n].fit_model(image_sets_dsa[n],label_sets_dsa[n],x_val,y_val)
n = n+1
n=15
print(n)

models_dsa[n].fit_model(image_sets_dsa[n],label_sets_dsa[n],x_val,y_val)
n = n+1
imporsleepepeptime
time.sleep(400)
print(n)

models_dsa[n].fit_model(image_sets_dsa[n],label_sets_dsa[n],x_val,y_val)
n = n+1

time.sleep(400)
print(n)

models_dsa[n].fit_model(image_sets_dsa[n],label_sets_dsa[n],x_val,y_val)
n = n+1
print(n)

models_dsa[n].fit_model(image_sets_dsa[n],label_sets_dsa[n],x_val,y_val)
n = n+1
n=19
print(n)#19

models_dsa[n].fit_model(image_sets_dsa[n],label_sets_dsa[n],x_val,y_val)
n = n+1
loading = True

models_dsa = []

if loading:
    for i in range(20):
        model_dsa_dir = "D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_dsa_e2_"+str(i)
        print(model_dsa_dir)
        model =utils.My_model('gtsrb',True,model_dsa_dir)
        model.model.compile(loss= 'categorical_crossentropy', optimizer = 'rmsprop', metrics = ['accuracy',tf.keras.metrics.Precision(), tf.keras.metrics.Recall()])
        models_dsa.append(model)<jupyter_output>D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_dsa_e2_0
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_dsa_e2_1
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_dsa_e2_2
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_dsa_e2_3
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_dsa_e2_4
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_dsa_e2_5
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_dsa_e2_6
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_dsa_e2_7
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_dsa_e2_8
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_dsa_e2_9
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_dsa_e2_10
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_dsa_e2_11
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_dsa_e2_12
Model loaded correc[...]<jupyter_text>Training guided by Random values<jupyter_code>import random
random_indexes =list(range(len(x_train_and_adversary)))
random.shuffle(random_indexes)
print(random_indexes[:10])
print(len(random_indexes))
save_dir = "C:/Users/fjdur/Desktop/upc/project_notebooks/github_project/DL_notebooks/SA_values/gtsrb_random_values_e2.npy"

#np.save(save_dir,np.array(random_indexes))

random_indexes = np.load(save_dir)
# Obtaining top n images by random values
top_images_by_random = utils.get_x_of_indexes(list(np.flip(np.argsort(random_indexes))),x_train_and_adversary)
top_labels_by_random = utils.get_x_of_indexes(list(np.flip(np.argsort(random_indexes))),y_train_and_adversary)
m = 1800
n = 0
image_sets_random = []
label_sets_random = []


for i in range(len(top_images_by_random)//m):
    print(i,":")
    if (i+1 >= len(top_images_by_random)//m):
        print("Last")
        print(0," -> ",n+m+(len(top_images_by_random)%m))
        top_images_by_random_n = np.array(top_images_by_random[:n+m+(len(top_images_by_random)%m)])
        top_labels_by_random_n = np.array(top_labels_by_random[:n+m+(len(top_images_by_random)%m)])
    else:
        print(0," -> ",m+n)
        top_images_by_random_n = np.array(top_images_by_random[:n+m])
        top_labels_by_random_n = np.array(top_labels_by_random[:n+m])
    image_sets_random.append(top_images_by_random_n)
    label_sets_random.append(top_labels_by_random_n)
    print(len(top_images_by_random_n))
    n += m


len(image_sets_random)
print(model_dir)

models_random = []
for i in range(len(label_sets_random)):
    print(i,":")
    model = utils.My_model('gtsrb',False,None)
    model.compile_model()
    models_random.append(model)

n=0
print(n)

models_random[n].fit_model(image_sets_random[n],label_sets_random[n],x_val,y_val)
n = n+1
print(n)

models_random[n].fit_model(image_sets_random[n],label_sets_random[n],x_val,y_val)
n = n+1
print(n)

models_random[n].fit_model(image_sets_random[n],label_sets_random[n],x_val,y_val)
n = n+1
print(n)

models_random[n].fit_model(image_sets_random[n],label_sets_random[n],x_val,y_val)
n = n+1
print(n)#4

models_random[n].fit_model(image_sets_random[n],label_sets_random[n],x_val,y_val)
n = n+1
print(n)

models_random[n].fit_model(image_sets_random[n],label_sets_random[n],x_val,y_val)
n = n+1
print(n)

models_random[n].fit_model(image_sets_random[n],label_sets_random[n],x_val,y_val)
n = n+1
n=7
print(n)

models_random[n].fit_model(image_sets_random[n],label_sets_random[n],x_val,y_val)
n = n+1
print(n)

models_random[n].fit_model(image_sets_random[n],label_sets_random[n],x_val,y_val)
n = n+1
n=9
print(n)#9

models_random[n].fit_model(image_sets_random[n],label_sets_random[n],x_val,y_val)
n = n+1
n=10
print(n)

models_random[n].fit_model(image_sets_random[n],label_sets_random[n],x_val,y_val)
n = n+1
print(n)

models_random[n].fit_model(image_sets_random[n],label_sets_random[n],x_val,y_val)
n = n+1
n=12
print(n)

models_random[n].fit_model(image_sets_random[n],label_sets_random[n],x_val,y_val)
n = n+1
print(n)

models_random[n].fit_model(image_sets_random[n],label_sets_random[n],x_val,y_val)
n = n+1
print(n)#14

models_random[n].fit_model(image_sets_random[n],label_sets_random[n],x_val,y_val)
n = n+1
print(n)

models_random[n].fit_model(image_sets_random[n],label_sets_random[n],x_val,y_val)
n = n+1
n=16
print(n)

models_random[n].fit_model(image_sets_random[n],label_sets_random[n],x_val,y_val)
n = n+1
print(n)

models_random[n].fit_model(image_sets_random[n],label_sets_random[n],x_val,y_val)
n = n+1
n=18
print(n)

models_random[n].fit_model(image_sets_random[n],label_sets_random[n],x_val,y_val)
n = n+1
n=19
print(n)#19

models_random[n].fit_model(image_sets_random[n],label_sets_random[n],x_val,y_val)
n = n+1
loading = True

models_random = []

if loading:
    for i in range(20):
        model_random_dir = "D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_random_e2_"+str(i)
        print(model_random_dir)
        model =utils.My_model('gtsrb',True,model_random_dir)
        model.model.compile(loss= 'categorical_crossentropy', optimizer = 'rmsprop', metrics = ['accuracy',tf.keras.metrics.Precision(), tf.keras.metrics.Recall()])
        models_random.append(model)<jupyter_output>D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_random_e2_0
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_random_e2_1
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_random_e2_2
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_random_e2_3
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_random_e2_4
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_random_e2_5
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_random_e2_6
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_random_e2_7
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_random_e2_8
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_random_e2_9
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_random_e2_10
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_random_e2_11
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_[...]<jupyter_text>Training guided by NC<jupyter_code># NC
nc_values = []
for i in range(1,38):
    save_dir_rand = "C:/Users/fjdur/Desktop/upc/project_notebooks/github_project/DL_notebooks/NC_values/nc_values_"+str(i)+".npy"
    #print(save_dir_rand)
    tmp_values = np.load(save_dir_rand)
    #print(tmp_values.shape)
    nc_values = np.append(nc_values,tmp_values)
nc_values.shape

top_images_by_nc = utils.get_x_of_indexes(list(np.flip(np.argsort(nc_values))),x_train_and_adversary)
top_labels_by_nc = utils.get_x_of_indexes(list(np.flip(np.argsort(nc_values))),y_train_and_adversary)

m = 1800
n = 0
image_sets_nc = []
label_sets_nc = []


for i in range(len(top_images_by_nc)//m):
    print(i,":")
    if (i+1 >= len(top_images_by_nc)//m):
        print("Last")
        print(0," -> ",n+m+(len(top_images_by_nc)%m))
        top_images_by_nc_n = np.array(top_images_by_nc[:n+m+(len(top_images_by_nc)%m)])
        top_labels_by_nc_n = np.array(top_labels_by_nc[:n+m+(len(top_images_by_nc)%m)])
    else:
        print(0," -> ",m+n)
        top_images_by_nc_n = np.array(top_images_by_nc[:n+m])
        top_labels_by_nc_n = np.array(top_labels_by_nc[:n+m])
    image_sets_nc.append(top_images_by_nc_n)
    label_sets_nc.append(top_labels_by_nc_n)
    print(len(top_images_by_nc_n))
    n += m


print(model_dir)

models_nc = []
for i in range(len(label_sets_nc)):
    print(i,":")
    model = utils.My_model('gtsrb',False,None)
    model.compile_model()
    models_nc.append(model)

n=0
print(n)

models_nc[n].fit_model(image_sets_nc[n],label_sets_nc[n],x_val,y_val)
n = n+1

print(n)

models_nc[n].fit_model(image_sets_nc[n],label_sets_nc[n],x_val,y_val)
n = n+1

print(n)

models_nc[n].fit_model(image_sets_nc[n],label_sets_nc[n],x_val,y_val)
n = n+1

print(n)

models_nc[n].fit_model(image_sets_nc[n],label_sets_nc[n],x_val,y_val)
n = n+1
#4
print(n)

models_nc[n].fit_model(image_sets_nc[n],label_sets_nc[n],x_val,y_val)
n = n+1
n=5
print(n)

models_nc[n].fit_model(image_sets_nc[n],label_sets_nc[n],x_val,y_val)
n = n+1
n=6
print(n)

models_nc[n].fit_model(image_sets_nc[n],label_sets_nc[n],x_val,y_val)
n = n+1
n=7
print(n)

models_nc[n].fit_model(image_sets_nc[n],label_sets_nc[n],x_val,y_val)
n = n+1
n=8
print(n)

models_nc[n].fit_model(image_sets_nc[n],label_sets_nc[n],x_val,y_val)
n = n+1
#reiniciar la 9
print(n)#9

models_nc[n].fit_model(image_sets_nc[n],label_sets_nc[n],x_val,y_val)
n = n+1
print(n)

models_nc[n].fit_model(image_sets_nc[n],label_sets_nc[n],x_val,y_val)
n = n+1
print(n)

models_nc[n].fit_model(image_sets_nc[n],label_sets_nc[n],x_val,y_val)
n = n+1
print(n)

models_nc[n].fit_model(image_sets_nc[n],label_sets_nc[n],x_val,y_val)
n = n+1
n=13
print(n)

models_nc[n].fit_model(image_sets_nc[n],label_sets_nc[n],x_val,y_val)
n = n+1
#14
n=14
print(n)

models_nc[n].fit_model(image_sets_nc[n],label_sets_nc[n],x_val,y_val)
n = n+1
time.sleep(180)

print(n)

models_nc[n].fit_model(image_sets_nc[n],label_sets_nc[n],x_val,y_val)
n = n+1
time.sleep(180)
n=16
print(n)

models_nc[n].fit_model(image_sets_nc[n],label_sets_nc[n],x_val,y_val)
n = n+1
n=17
print(n)

models_nc[n].fit_model(image_sets_nc[n],label_sets_nc[n],x_val,y_val)
n = n+1 
print(n)

models_nc[n].fit_model(image_sets_nc[n],label_sets_nc[n],x_val,y_val)
n = n+1

models_nc[19] = utils.My_model('gtsrb',True,model_dir)
models_nc[19].compile_model()
n=19
print(n)

models_nc[n].fit_model(image_sets_nc[n],label_sets_nc[n],x_val,y_val)
n = n+1
loading = True

models_nc = []

if loading:
    for i in range(20):
        model_nc_dir = "D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_nc_e2_"+str(i)
        print(model_nc_dir)
        model =utils.My_model('gtsrb',True,model_nc_dir)
        model.model.compile(loss= 'categorical_crossentropy', optimizer = 'rmsprop', metrics = ['accuracy',tf.keras.metrics.Precision(), tf.keras.metrics.Recall()])
        models_nc.append(model)<jupyter_output>D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_nc_e2_0
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_nc_e2_1
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_nc_e2_2
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_nc_e2_3
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_nc_e2_4
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_nc_e2_5
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_nc_e2_6
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_nc_e2_7
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_nc_e2_8
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_nc_e2_9
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_nc_e2_10
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_nc_e2_11
Model loaded correctly
D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_nc_e2_12
Model loaded correctly
D:/models[...]<jupyter_text>Evaluating<jupyter_code>model_original.model.compile(loss= 'categorical_crossentropy', optimizer = 'rmsprop', metrics = ['accuracy',tf.keras.metrics.Precision(), tf.keras.metrics.Recall()])
# Metrics using adversarial test
evaluate_lsa_0 = []
evaluate_dsa_0 = []
evaluate_nc_0 = []
evaluate_random_0 = []

evaluate_lsa_0.append(model_original.evaluate(x_adversary_test_fgsm,y_adversary_test_fgsm))
evaluate_dsa_0.append(model_original.evaluate(x_adversary_test_fgsm,y_adversary_test_fgsm))
evaluate_nc_0.append(model_original.evaluate(x_adversary_test_fgsm,y_adversary_test_fgsm))
evaluate_random_0.append(model_original.evaluate(x_adversary_test_fgsm,y_adversary_test_fgsm))

for model in models_lsa:
    evaluate_lsa_0.append(model.evaluate(x_adversary_test_fgsm,y_adversary_test_fgsm))
    
for model in models_dsa:
    evaluate_dsa_0.append(model.evaluate(x_adversary_test_fgsm,y_adversary_test_fgsm))
    
for model in models_random:
    evaluate_random_0.append(model.evaluate(x_adversary_test_fgsm,y_adversary_test_fgsm))
    
for model in models_nc:
    evaluate_nc_0.append(model.evaluate(x_adversary_test_fgsm,y_adversary_test_fgsm))
evaluate_lsa_0[0]=[0,0,0,0]
evaluate_dsa_0[0]=[0,0,0,0]
evaluate_random_0[0]=[0,0,0,0]
evaluate_nc_0[0]=[0,0,0,0]
print(len(evaluate_lsa_0))
print(len(evaluate_dsa_0))
print(len(evaluate_random_0))
print(len(evaluate_nc_0))
# Metrics using original test
evaluate_lsa_1 = []
evaluate_dsa_1 = []
evaluate_nc_1 = []
evaluate_random_1 = []

evaluate_lsa_1.append(model_original.evaluate(x_test,y_test))
evaluate_dsa_1.append(model_original.evaluate(x_test,y_test))
evaluate_nc_1.append(model_original.evaluate(x_test,y_test))
evaluate_random_1.append(model_original.evaluate(x_test,y_test))
print("lsa---")
for model in models_lsa:
    evaluate_lsa_1.append(model.evaluate(x_test,y_test))

print("dsa---")
    
for model in models_dsa:
    evaluate_dsa_1.append(model.evaluate(x_test,y_test))
    

print("random---")

for model in models_random:
    evaluate_random_1.append(model.evaluate(x_test,y_test))
    
print("nc---")

for model in models_nc:
    evaluate_nc_1.append(model.evaluate(x_test,y_test))

evaluate_lsa_1[0]=[0,0,0,0]
evaluate_dsa_1[0]=[0,0,0,0]
evaluate_random_1[0]=[0,0,0,0]
evaluate_nc_1[0]=[0,0,0,0]

print(len(evaluate_lsa_1))
print(len(evaluate_dsa_1))
print(len(evaluate_random_1))
print(len(evaluate_nc_1))
type(evaluate_dsa_0[0][0])
import pandas as pd

df_evaluate_lsa_0 = pd.DataFrame(np.array(evaluate_lsa_0),columns=["loss","accuracy","precision","recall"])
df_evaluate_lsa_1 = pd.DataFrame(np.array(evaluate_lsa_1),columns=["loss","accuracy","precision","recall"])
df_evaluate_dsa_0 = pd.DataFrame(np.array(evaluate_dsa_0),columns=["loss","accuracy","precision","recall"])
df_evaluate_dsa_1 = pd.DataFrame(np.array(evaluate_dsa_1),columns=["loss","accuracy","precision","recall"])

df_evaluate_random_0 = pd.DataFrame(np.array(evaluate_random_0),columns=["loss","accuracy","precision","recall"])
df_evaluate_random_1 = pd.DataFrame(np.array(evaluate_random_1),columns=["loss","accuracy","precision","recall"])
df_evaluate_nc_0 = pd.DataFrame(np.array(evaluate_nc_0),columns=["loss","accuracy","precision","recall"])
df_evaluate_nc_1 = pd.DataFrame(np.array(evaluate_nc_1),columns=["loss","accuracy","precision","recall"])

# Original test set + adversarial set

metric ="accuracy"
accuracy_lsa_3 = (np.array(df_evaluate_lsa_0[metric])+np.array(df_evaluate_lsa_1[metric]))/2
accuracy_dsa_3 = (np.array(df_evaluate_dsa_0[metric])+np.array(df_evaluate_dsa_1[metric]))/2
accuracy_nc_3 = (np.array(df_evaluate_nc_0[metric])+np.array(df_evaluate_nc_1[metric]))/2
accuracy_random_3 = (np.array(df_evaluate_random_0[metric])+np.array(df_evaluate_random_1[metric]))/2<jupyter_output><empty_output><jupyter_text>Charts<jupyter_code>n_inputs = [1800*i for i in range(20)]
n_inputs.append(len(x_train_and_adversary))
print(n_inputs)
cd "C:/Users/fjdur/Desktop/gtsrb_graphs/e2_graphs_dec/"
linestyles = ['solid','dotted','dashed','dashdot']
colors =['k','k','k','k']
#configuration 1
"""
6) Incremental guided retraining starting from the original model using only the new adversarial inputs.
Incremental training, starting with the previous trained model. Using at each iteration a subset of the new inputs.
"""
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
from matplotlib.patches import Patch
from matplotlib.lines import Line2D

#metric = "accuracy" # accuracy loss
my_metrics =["accuracy","loss","precision","recall"]

for metric in my_metrics:
    plt.clf()
    plt.plot(n_inputs,df_evaluate_lsa_0[metric],colors[0],linestyle=linestyles[0])
    plt.plot(n_inputs,df_evaluate_dsa_0[metric],colors[1],linestyle=linestyles[1])
    plt.plot(n_inputs,df_evaluate_random_0[metric],colors[2],linestyle=linestyles[2])
    plt.plot(n_inputs,df_evaluate_nc_0[metric],colors[3],linestyle=linestyles[3])


    legend_elements = [Line2D([0], [0], color='k', label='LSA',ls = linestyles[0]),
                   Line2D([0], [0], color='k', label='DSA',ls = linestyles[1]),
                   Line2D([0], [0], color='k', label='Random',ls = linestyles[2]),
                   Line2D([0], [0], color='k', label='NC',ls = linestyles[3])]



    plt.legend(handles=legend_elements)#
    plt.title(metric + " with test set of adversarial examples FGSM")
    plt.xlim([0, 37000])
    #plt.ylim([0, 1])
    plt.xlabel('number of inputs')
    plt.ylabel(metric)
    plt.savefig("gtsrb_c1_"+metric + "_0.svg")
    plt.show()
#configuration 1
"""
6) Incremental guided retraining starting from the original model using only the new adversarial inputs.
Incremental training, starting with the previous trained model. Using at each iteration a subset of the new inputs.
"""
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches

my_metrics =["accuracy","loss","precision","recall"]

for metric in my_metrics:
    
#metric = "accuracy" # accuracy loss
    plt.clf()
    plt.plot(n_inputs,df_evaluate_lsa_1[metric],colors[0],linestyle=linestyles[0])
    plt.plot(n_inputs,df_evaluate_dsa_1[metric],colors[1],linestyle=linestyles[1])
    plt.plot(n_inputs,df_evaluate_random_1[metric],colors[2],linestyle=linestyles[2])
    plt.plot(n_inputs,df_evaluate_nc_1[metric],colors[3],linestyle=linestyles[3])

    legend_elements = [Line2D([0], [0], color='k', label='LSA',ls = linestyles[0]),
                   Line2D([0], [0], color='k', label='DSA',ls = linestyles[1]),
                   Line2D([0], [0], color='k', label='Random',ls = linestyles[2]),
                   Line2D([0], [0], color='k', label='NC',ls = linestyles[3])]



    plt.legend(handles=legend_elements)#
    plt.title(metric + " with original set")
    plt.xlim([0, 37000])
    #plt.ylim([0, 1])
    plt.xlabel('number of inputs')
    plt.ylabel(metric)
    plt.savefig("gtsrb_c1_"+metric + "_1.svg")
    plt.show()
#adversarial jsma test set
#configuration 3
"""
6) Incremental guided retraining starting from the original model using only the new adversarial inputs.
Incremental training, starting with the previous trained model. Using at each iteration a subset of the new inputs.
"""
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches

plt.plot(n_inputs,accuracy_lsa_3,colors[0],linestyle=linestyles[0])
plt.plot(n_inputs,accuracy_dsa_3,colors[1],linestyle=linestyles[1])
plt.plot(n_inputs,accuracy_random_3,colors[2],linestyle=linestyles[2])
plt.plot(n_inputs,accuracy_nc_3,colors[3],linestyle=linestyles[3])


legend_elements = [Line2D([0], [0], color='k', label='LSA',ls = linestyles[0]),
                   Line2D([0], [0], color='k', label='DSA',ls = linestyles[1]),
                   Line2D([0], [0], color='k', label='Random',ls = linestyles[2]),
                   Line2D([0], [0], color='k', label='NC',ls = linestyles[3])]



plt.legend(handles=legend_elements)#
plt.title("accuracy with both sets")

print(n_inputs[np.argmax(accuracy_lsa_3)],accuracy_lsa_3.max())
print(n_inputs[np.argmax(accuracy_dsa_3)],accuracy_dsa_3.max())
print(n_inputs[np.argmax(accuracy_random_3)],accuracy_random_3.max())
print(n_inputs[np.argmax(accuracy_nc_3)],accuracy_nc_3.max())

plt.plot(n_inputs[np.argmax(accuracy_lsa_3)],accuracy_lsa_3.max(),'-kD')
plt.plot(n_inputs[np.argmax(accuracy_dsa_3)],accuracy_dsa_3.max(),'-ko')
plt.plot(n_inputs[np.argmax(accuracy_random_3)],accuracy_random_3.max(),'-kv')
plt.plot(n_inputs[np.argmax(accuracy_nc_3)],accuracy_nc_3.max(),'-kp')

plt.xlabel('number of inputs')
plt.ylabel('accuracy')
plt.xlim([0, 37000])
#plt.ylim([0, 1])

plt.savefig("gtsrb_c1_"+"accuracy" + "_both.svg")

plt.show()<jupyter_output>36366 0.9668620824813843
36366 0.9653326570987701
27000 0.966352254152298
36366 0.9629110395908356<jupyter_text>Saving models<jupyter_code>new_model_lsa_dir  = "D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_lsa_e2"
i=0

for model in models_lsa:
    model.save(new_model_lsa_dir+"_"+str(i))
    i+=1
new_model_dsa_dir  = "D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_dsa_e2"

i=0
for model in models_dsa:
    model.save(new_model_dsa_dir+"_"+str(i))
    i+=1
new_model_random_dir  = "D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_random_e2"

i=0
for model in models_random:
    model.save(new_model_random_dir+"_"+str(i))
    i+=1
new_model_nc_dir  = "D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_nc_e2"

i=0
for model in models_nc:
    model.save(new_model_nc_dir+"_"+str(i))
    i+=1<jupyter_output>INFO:tensorflow:Assets written to: D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_nc_e2_0\assets
Model has been saved
INFO:tensorflow:Assets written to: D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_nc_e2_1\assets
Model has been saved
INFO:tensorflow:Assets written to: D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_nc_e2_2\assets
Model has been saved
INFO:tensorflow:Assets written to: D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_nc_e2_3\assets
Model has been saved
INFO:tensorflow:Assets written to: D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_nc_e2_4\assets
Model has been saved
INFO:tensorflow:Assets written to: D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_nc_e2_5\assets
Model has been saved
INFO:tensorflow:Assets written to: D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_nc_e2_6\assets
Model has been saved
INFO:tensorflow:Assets written to: D:/models/gtsrb_models/C1/gtsrb_model_c1_sep_nc_e2_7\assets
Model has been saved
INFO:tensorflow:Assets written to: D:/models/gtsrb_models/C1/gtsrb_model[...]
{'fraction_non_alphanumeric': 0.13797991002649904, 'fraction_numerical': 0.034233068342885316, 'mean_word_length': 7.103620474406991, 'pattern_counts': {'":': 8, '<': 32, '<?xml version=': 0, '>': 64, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '542421', 'license': 'MIT', 'n_tokens_mistral': 14861, 'n_tokens_neox': 14266, 'n_words': 1689, 'path': 'notebooks/3-4.0.1-gtsrb-ordering_and_retraining-configuration_1.ipynb', 'repo_name': 'fjdurlop/guided-retraining', 'size': '32454'}