<jupyter_start><jupyter_text><jupyter_code>!pip install 'h5py==2.10.0' --force-reinstall

!pip uninstall keras-nightly
!pip uninstall -y tensorflow
!pip install tensorflow_gpu==2.4.0
!pip install keras==2.4.0

!pip install keras --user
#Keras preprosessing issue
!pip uninstall keras-preprocessing
!pip install git+https://github.com/keras-team/keras-preprocessing.git
!pip install np_utils
import pandas as pd
import numpy as np
pd.set_option('display.max_columns', 50)
from scipy.spatial import cKDTree
np.random.seed(123)
from shapely.ops import nearest_points
from math import *
import urllib, os
import matplotlib.pyplot as plt
%matplotlib inline
import datetime
from tensorflow.keras.preprocessing.image import ImageDataGenerator, array_to_img, img_to_array, load_img
from sklearn.preprocessing import MinMaxScaler
import glob
import cv2
import imageio
import imgaug as ia
from tensorflow.keras.models import Model
from keras.layers.normalization import BatchNormalization
from keras.layers.convolutional import Conv2D, MaxPooling2D
from keras.layers.core import Activation, Dropout, Dense
from keras.layers import Flatten, Input, concatenate,ZeroPadding2D
#from keras import Sequential
from tensorflow.keras.layers import Conv2D, Flatten, Dense

from sklearn.model_selection import train_test_split
from tensorflow.keras.optimizers import Adam, RMSprop
import argparse
import locale

from tensorflow.keras.preprocessing.image import ImageDataGenerator
from sklearn.metrics import confusion_matrix, classification_report,ConfusionMatrixDisplay
import itertools
from tensorflow.keras import regularizers
from IPython.display import SVG
from keras.utils.vis_utils import model_to_dot
#from keras.utils import plot_model
import imgaug.augmenters as iaa
from tensorflow.keras.models import load_model
from tensorflow.python.keras import backend as k
from tensorflow.keras.models import Sequential, load_model
from tensorflow.keras.layers import LSTM, Dense, RepeatVector, Masking, TimeDistributed
from tensorflow. keras.utils import plot_model
from tensorflow.keras.layers.experimental import preprocessing
from tensorflow.keras.applications import EfficientNetB4
from tensorflow.keras.applications import ResNet101
from tensorflow.keras import layers
import tensorflow as tf
import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()
tf.compat.v1.get_default_graph()
from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint, LearningRateScheduler
from google.colab import drive
drive.mount('/content/drive/')
def create_cnn(width, height, depth, filters=(16, 32, 64,128,256), regularizer=None):

    # Initialize the input shape and channel dimension, where the number of channels is the last dimension
    inputShape = (height, width, depth)
    chanDim = -1
 
    # Define the model input
    inputs = Input(shape=inputShape)
 
    # Loop over the number of filters 
    for (i, f) in enumerate(filters):
        # If this is the first CONV layer then set the input appropriately
        if i == 0:
            x = inputs
 
        # Create loops of CONV => RELU => BN => POOL layers
        x = Conv2D(f, (3, 3), padding="same")(x)
        x = Activation("relu")(x)
        x = BatchNormalization(axis=chanDim)(x)
        x = MaxPooling2D(pool_size=(2, 2))(x)
        
    # Final layers - flatten the volume, then Fully-Connected => RELU => BN => DROPOUT
    x = Flatten()(x)
    x = Dense(16, kernel_regularizer=regularizer)(x)
    x = Activation("relu")(x)
    x = BatchNormalization(axis=chanDim)(x)
    x = Dropout(0.2)(x)
 
    # Apply another fully-connected layer, this one to match the number of nodes coming out of the MLP
    x = Dense(3, kernel_regularizer=regularizer)(x)
    x = Activation("relu")(x)
    x = Dense(3, activation="softmax", name='last_layer')(x)
 
    # Construct the CNN
    model = Model(inputs, x)
 
    # Return the CNN
    return model  
def add_jpg(df):

  name_images = []
  for index, row in df.iterrows():
    name_images.append(row['name']+".jpg")
  df['name'] = name_images
df_train  =  pd.read_csv('/content/drive/My Drive/train/df_train')
train_dir = '/content/drive/My Drive/train'
test_dir = '/content/drive/My Drive/test'
df_test =  pd.read_csv('/content/drive/My Drive/test/df_test')


df_train.reset_index(drop=True, inplace=True)

df_test.reset_index(drop=True, inplace=True)

df_train = df_train.loc[:, ~df_train.columns.str.contains('^Unnamed')]
df_test = df_test.loc[:, ~df_test.columns.str.contains('^Unnamed')]
add_jpg(df_train)
add_jpg(df_test)
df_train['meta.clinical.diagnosis'] = df_train['meta.clinical.diagnosis'].astype('str')
df_test['meta.clinical.diagnosis'] = df_test['meta.clinical.diagnosis'].astype('str')
from keras.preprocessing.image import ImageDataGenerator
with tf.device('/device:GPU:0'):
    train_datagen = ImageDataGenerator(
      rescale=1.0/255.0,
      validation_split=0.20,
      rotation_range=40,
      width_shift_range=0.2,
      height_shift_range=0.2,
      shear_range=0.2,
      zoom_range=0.2,
      horizontal_flip=True,
      fill_mode='nearest'

    )

    train_generator = train_datagen.flow_from_dataframe(
      dataframe = df_train,
      directory = train_dir,
      x_col="name",
      y_col="meta.clinical.diagnosis",
      target_size=(512,512),
      subset="training",
      batch_size=16,
      seed =42,
      shuffle=True,
      class_mode="categorical"
    )

    val_generator = train_datagen.flow_from_dataframe(
      dataframe = df_train,
      directory = train_dir,
      x_col="name",
      y_col="meta.clinical.diagnosis",
      target_size=(512,512),
      subset="validation",
      batch_size=16,
      seed = 42,
      shuffle=True,
      class_mode="categorical"
    )  
    
cnn1 = create_cnn(512,512,3,regularizer=regularizers.l2(0.001))
cnn1.summary()
start = datetime.datetime.now()
# compile the model 
optimizer = tf.keras.optimizers.Adam(learning_rate=0.001)
cnn1.compile(optimizer=optimizer, loss="categorical_crossentropy", metrics=["accuracy"])

earlystopping = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=6) 

checkpointer = ModelCheckpoint(filepath="/content/drive/My Drive/Cnn_lr_0.001_16.h5", verbose=1, save_best_only=True)

model_history = cnn1.fit( train_generator, steps_per_epoch= train_generator.n // 16, epochs = 200, validation_data= val_generator, validation_steps= val_generator.n // 16, callbacks=[checkpointer, earlystopping])
end = datetime.datetime.now()
print("Time taken to run:", end-start)
def model_evaluation(model, history,  train_images, train_labels, test_features, test_images, test_labels, class_names=None, model_name=None):
    """
    Evaluates the performance of a CNN with loss and accuracy plots, a confusion matrix and a classification report for the training and test sets.
    """
    train_acc = history.history['acc']
    val_acc = history.history['val_acc']
    train_loss = history.history['loss']
    val_loss = history.history['val_loss']
    epch = range(1, len(train_acc) + 1)
    plt.plot(epch, train_acc, 'g.', label='Training Accuracy')
    plt.plot(epch, val_acc, 'g', label='Validation acc')
    plt.title('Accuracy')
    plt.legend()
    plt.figure()
    plt.plot(epch, train_loss, 'r.', label='Training loss')
    plt.plot(epch, val_loss, 'r', label='Validation loss')
    plt.title('Loss')
    plt.legend()
    plt.show()
    
    #results_test = model.evaluate([test_features, test_images], test_labels)
    #print('Test Loss:', results_test[0])
    #print('Test Accuracy:', results_test[1])
    
    y_train_pred = np.round(model.predict([train_features, train_images]))
    y_pred = np.round(model.predict([test_features, test_images]))
    
    
    print(classification_report(train_labels, y_train_pred,target_names=class_names))
    print(classification_report(test_labels, y_pred,target_names=class_names))
    confusion_matrix_plot(test_labels,y_pred)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10947341899722852, 'fraction_numerical': 0.017006802721088437, 'mean_word_length': 4.606638418079096, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '166991', 'license': 'MIT', 'n_tokens_mistral': 2710, 'n_tokens_neox': 2566, 'n_words': 598, 'path': 'CNN.ipynb', 'repo_name': 'yemre-data/DeepLearning_classification_skinlession_detection', 'size': '7938'}