<jupyter_start><jupyter_text>dataframe<jupyter_code>colunas = ['treinamento', 'teste'] #criando as colunas

df = pd.DataFrame(data=data_train_labels, columns = colunas)<jupyter_output><empty_output><jupyter_text>Validação do modelo | Kfold<jupyter_code>idg = ImageDataGenerator(width_shift_range=0.1,
                         height_shift_range=0.1,
                         zoom_range=0.3,
                         fill_mode='nearest',
                         horizontal_flip = True,
                         rescale=1./255)

def get_model_name(k):
    return 'model_'+str(k)+'.h5'
df['treinamento'] = df['treinamento'].astype(str)
df['teste']= df['teste'].astype(str)

#print(type(df.treinamento))
#print(type(df.teste))
x = df['treinamento']
Y = df[['teste']]
save_dir = '/content/drive/MyDrive/Colab Notebooks/teste/saved_models/'


fold_var = 1

kfold = KFold(5, True, 1)

for train, test in kfold.split(np.zeros(x.shape[0]), Y):

  training_data = df.iloc[train]
  validation_data = df.iloc[test]
  #print('validation_data')
  #print(validation_data.size)

  train_data_generator = idg.flow_from_dataframe(training_data,directory=None,
		x_col = "treinamento",
    y_col = "teste",
    class_mode = "categorical",
    target_size=(224, 224),
    color_mode="rgb",
    shuffle = True)
  
  valid_data_generator  = idg.flow_from_dataframe(validation_data,directory=None,
    x_col = "treinamento",
    y_col = "teste",
    class_mode = "categorical",
    target_size=(224, 224),
    color_mode="rgb",
    shuffle = True)


  #Instanciando o modelo VGG16
  vgg = keras.applications.vgg16.VGG16(
    input_shape = (224,224,3),
    weights = 'imagenet', #pesos pré-treinados do imagenet (transfer-learning workflow)
    include_top = False
  )

  #vgg.input
  vgg.output

  #"congelar" o modelo básico
  for layer in vgg.layers:
    layer.trainable = False

  x = keras.layers.Flatten()(vgg.output)
  prediction = keras.layers.Dense(2, activation='softmax')(x) #duas classes (B e M)
  my_model = keras.Model(inputs=vgg.input, outputs=prediction)
  #my_model.summary()

  adam = keras.optimizers.Adam()

  #compilando o model
  my_model.compile(
      loss='categorical_crossentropy',
		  optimizer=adam,
		  metrics=['accuracy']
      )
  
  #treinamento da rede
  #*********************************************************
  '''
  checkpoint = tf.keras.callbacks.ModelCheckpoint(save_dir+get_model_name(fold_var),
                                                  monitor='val_accuracy', verbose=1,
                                                  save_best_only=True, mode='max')

  callbacks_list = [checkpoint]
  
  history = my_model.fit(train_data_generator,
                              validation_data=valid_data_generator,
                              epochs = 6,#6
                              verbose=1,
                              callbacks=callbacks_list)

  #print(my_model_fit)
  my_model.load_weights("/content/drive/MyDrive/Colab Notebooks/teste/saved_models/model_"+str(fold_var)+".h5")

  results = my_model.evaluate(valid_data_generator)
  '''
  #*********************************************************

  tf.keras.backend.clear_session()

  fold_var += 1<jupyter_output>Found 93 validated image filenames belonging to 2 classes.
Found 24 validated image filenames belonging to 2 classes.
Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/vgg16/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5
58892288/58889256 [==============================] - 0s 0us/step
58900480/58889256 [==============================] - 0s 0us/step
Found 93 validated image filenames belonging to 2 classes.
Found 24 validated image filenames belonging to 2 classes.
Found 94 validated image filenames belonging to 2 classes.
Found 23 validated image filenames belonging to 2 classes.
Found 94 validated image filenames belonging to 2 classes.
Found 23 validated image filenames belonging to 2 classes.
Found 94 validated image filenames belonging to 2 classes.
Found 23 validated image filenames belonging to 2 classes.<jupyter_text>Testando uma imagem (M)<jupyter_code>img = keras.preprocessing.image.load_img("/content/drive/MyDrive/Colab Notebooks/mdb270.PNG",target_size=(224,224)) #M
img = np.asarray(img)
plt.imshow(img)
img = np.expand_dims(img, axis=0)

#saved_model = keras.models.load_model("/content/drive/MyDrive/Colab Notebooks/teste/saved_models/model_1.h5")
#print(saved_model)

output = my_model.predict(img)

print('output')
print(output)

predictions = my_model.predict_generator(img)

#print('predictions')
#print(predictions)

print("\n**************************************\n")

argmax = np.argmax(predictions)
print(np.argmax(predictions)) #classe

if(argmax == 0):
    print('Benigno')

if(argmax == 1):
     print("Maligno")<jupyter_output>output
[[0.9491137  0.05088628]]<jupyter_text>Confusion matrix | Specificity and Sensitivity<jupyter_code>tam = valid_data_generator.classes

cont = 0
for i in tam:
  if(i == 0):
    cont+=1

#print(cont)
#Função que calcula uma confusion_matrix para todos os models gerados 

from sklearn.metrics import classification_report, confusion_matrix 
from sklearn.linear_model import LinearRegression
from sklearn.metrics import ConfusionMatrixDisplay

teste = []

def confusion_matrix_model(model, i):
  classes = [0,1]

  predict = model.predict(valid_data_generator)
  y_pred = np.argmax(predict, axis=1)
  y_true = valid_data_generator.classes
  cm = confusion_matrix(y_true, y_pred)
  
  teste.append(cm[0][0])
  teste.append(cm[1][1])

  cm_display = ConfusionMatrixDisplay(confusion_matrix=cm,display_labels= classes)
  cm_display.plot()
  plt.suptitle(i)

caminho = '/content/drive/MyDrive/Colab Notebooks/teste/saved_models'

j=0
sensibility_total =0
specificity_total=0

for i in os.listdir(caminho):
  m = '/content/drive/MyDrive/Colab Notebooks/teste/saved_models/{}'.format(i)
  m = keras.models.load_model(m)
  confusion_matrix_model(m, i)
    
  tam = len(valid_data_generator.filenames)

  #Sensibilidade = casos positivos corretos 
  sensibility = teste[j] / cont
  sensibility_total += sensibility

  #Especificidade = casos negativos corretos
  specificity =  teste[j+1] / (tam - cont)
  specificity_total += specificity

  print('{}\nSensibilidade = {} | Especificidade = {}\n'.format(i, sensibility, specificity))
  j+=2

#Sensibilidade e Especificidade total
print('\nSensibilidade Total = {} | Especificidade Total = {}\n'.format((sensibility_total / (j/2)), (specificity_total/ (j/2))))

#print(sensibility_total)
#print(specificity_total)<jupyter_output>model_1.h5
Sensibilidade = 0.0 | Especificidade = 1.0

model_2.h5
Sensibilidade = 1.0 | Especificidade = 0.0

model_3.h5
Sensibilidade = 0.875 | Especificidade = 0.0

WARNING:tensorflow:5 out of the last 6 calls to <function Model.make_predict_function.<locals>.predict_function at 0x7fbd0e9f5830> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has experimental_relax_shapes=True option that relaxes argument shapes that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for  more details.
model_4.h5
Sensibilidade = 0.9375 | Especificidade = 0.0

WARNING:tensorflow:6 out of the last 7 [...]<jupyter_text>Curva roc  https://www.flai.com.br/juscudilio/como-calcular-as-metricas-de-validacao-dos-modelos-de-machine-learning-em-python/<jupyter_code>import numpy as np
from sklearn import metrics
import matplotlib.pyplot

saved_model = keras.models.load_model("/content/drive/MyDrive/Colab Notebooks/teste/saved_models/model_1.h5")

predict = saved_model.predict(valid_data_generator)
y_pred = np.argmax(predict, axis=1)
y_true = valid_data_generator.classes

y_pred = y_pred.tolist() #convertendo para lista

fpr, tpr, thresholds = metrics.roc_curve(y_true, y_pred)
roc_auc  = metrics.auc(fpr, tpr)

plt.plot(fpr, tpr, label = 'AUC = %0.2f' % roc_auc)

#teste 1

import sklearn 

saved_model = keras.models.load_model("/content/drive/MyDrive/Colab Notebooks/teste/saved_models/model_1.h5")
y_test = valid_data_generator.classes

y_pred_probability = saved_model.predict(valid_data_generator)[::,1]
fpr, tpr, _ = metrics.roc_curve(y_test,  y_pred_probability)

auc = metrics.roc_auc_score(y_test, y_pred_probability)
plt.plot(fpr,tpr,label="data 1, auc="+str(auc))
plt.legend(loc=4)
plt.show()
#teste 2 | erro 

import sklearn 

saved_model = keras.models.load_model("/content/drive/MyDrive/Colab Notebooks/teste/saved_models/model_1.h5")
y_test = valid_data_generator.classes

y_pred_probability = saved_model.predict(valid_data_generator)
y_pred = np.argmax(y_pred_probability, axis=1)

fpr, tpr = metrics.roc_curve(y_test,y_pred)

auc = metrics.roc_auc_score(y_test, y_pred)

plt.plot(fpr,tpr,label="data 1, auc="+str(auc))
plt.legend(loc=4)
plt.show()
# teste 3 | erro ?
saved_model = keras.models.load_model("/content/drive/MyDrive/Colab Notebooks/teste/saved_models/model_1.h5")

predict = saved_model.predict(valid_data_generator)
y_pred = np.argmax(predict, axis=1)
y_true = valid_data_generator.classes

print("y_test", y_test)
print("y_pred", y_pred)

fpr, tpr, thresholds = metrics.roc_curve(y_true, y_pred, pos_label=2)
metrics.auc(fpr, tpr)<jupyter_output><empty_output><jupyter_text>Accuracy<jupyter_code># Accuracy de cada modelo | correto

from sklearn import metrics        
saved_model = keras.models.load_model("/content/drive/MyDrive/Colab Notebooks/teste/saved_models/model_1.h5")

predict = saved_model.predict(valid_data_generator)
y_pred = np.argmax(predict, axis=1)
y_test = valid_data_generator.classes

print("y_test", y_test)
print("y_pred", y_pred)

print("Accuracy", metrics.accuracy_score(y_test, y_pred))

'''
import numpy as np
from sklearn import metrics

y = np.array([1, 1, 2, 2])
pred = np.array([0.1, 0.4, 0.35, 0.8])

fpr, tpr, thresholds = metrics.roc_curve(y, pred, pos_label=2)
metrics.auc(fpr, tpr)
'''<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13542478565861263, 'fraction_numerical': 0.024551831644583008, 'mean_word_length': 4.643210555250137, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 4, 'lorem ipsum': 0, 'www.': 3, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '591199', 'license': 'MIT', 'n_tokens_mistral': 3817, 'n_tokens_neox': 3572, 'n_words': 693, 'path': 'CNN_kfold/cnn_kfold_v1.ipynb', 'repo_name': 'HeloiseKatharine/PIBIC', 'size': '10264'}