<jupyter_start><jupyter_text>**Set the path names (csv + image folder)**<jupyter_code>csv_label0 = "/content/gdrive/MyDrive/Colab Notebooks/dataset_label0_Inocybaceae.csv"
csv_label1 = "/content/gdrive/MyDrive/Colab Notebooks/dataset_label1_Omphalotaceae.csv"
csv_label2 = "/content/gdrive/MyDrive/Colab Notebooks/dataset_label2_Fomitopsidaceae.csv"
csv_label3 = "/content/gdrive/MyDrive/Colab Notebooks/dataset_label3_Physalacriaceae.csv"
csv_label4 = "/content/gdrive/MyDrive/Colab Notebooks/dataset_label4_Marasmiaceae.csv"


folderpath = "/content/gdrive/MyDrive/images_mushroom/"
df = pd.read_csv(csv_label2)
df['filepath'] = [folderpath + str(df['image_id'][x]) + ".jpg" for x in range(len(df))]
df.head()
# Just to run a simple test:

im = tf.io.read_file(df['filepath'].iloc[37])
# Décoder l'information en un tensorflow RGB (3 channels).
im = tf.image.decode_jpeg(im, channels=3)

plt.imshow(im);<jupyter_output><empty_output><jupyter_text>**Set the train/test datagenerators !**<jupyter_code>train_data_generator = ImageDataGenerator(rotation_range=5,
                                          width_shift_range = 0.1,
                                          height_shift_range = 0.05,
                                          zoom_range = 1.1,
                                          horizontal_flip = True,
                                          vertical_flip = True)

test_data_generator = ImageDataGenerator()<jupyter_output><empty_output><jupyter_text>**Modèle famille 1 --> label0**<jupyter_code>df = pd.read_csv(csv_label0)
df['filepath'] = [folderpath + str(df['image_id'][x]) + ".jpg" for x in range(len(df))]

display(df.head())

data_train, data_test = train_test_split(df, test_size=0.2, random_state=42)
data_train.head()

batch_size = 32

data_train["label"] = data_train["label"].astype(str)
data_test["label"] = data_test["label"].astype(str)

train_generator = train_data_generator.flow_from_dataframe(dataframe = data_train, directory="", x_col = "filepath", y_col="label", target_size=(256,256), batch_size=batch_size, class_mode='sparse')
test_generator = test_data_generator.flow_from_dataframe(dataframe = data_test, directory="", x_col = "filepath", y_col="label", target_size=(256,256), batch_size=batch_size, class_mode='sparse')

num_class = df["genus"].nunique()

efficientNet = EfficientNetB1(include_top=False, input_shape=(256, 256, 3))
for layer in efficientNet.layers[-1:]:
    layer.trainable = True

model = Sequential()
model.add(efficientNet)
model.add(GlobalAveragePooling2D())
model.add(Dense(units = 1024, activation = 'relu'))
model.add(Dropout(0.2))
model.add(Dense(units = 512, activation = 'relu'))
model.add(Dropout(0.2))
model.add(Dense(units = num_class, activation = 'softmax'))

model.compile(optimizer = 'Adam', loss = 'sparse_categorical_crossentropy', metrics = ['accuracy'])

lr_plateau = callbacks.ReduceLROnPlateau(monitor = 'val_loss',
                                         patience = 3,
                                         factor = 0.1,
                                         verbose = 2,
                                         mode = 'min')

early_stopping = callbacks.EarlyStopping(monitor = "val_loss",
                                         patience = 7,
                                         mode = 'min',
                                         verbose = 2,
                                         restore_best_weights= True)

checkpoint = callbacks.ModelCheckpoint(filepath="/content/gdrive/MyDrive/Colab Notebooks/modele_genus/checkpoint_genus_fam1/model_label0_effnetB1_temp_20210627.h5",
                                      monitor = 'val_loss',
                                      save_best_only = True,
                                      save_weights_only = False,
                                      mode = 'min',
                                      save_freq = 'epoch')


history = model.fit(train_generator, 
                              epochs = 40,
                              steps_per_epoch = len(data_train)//batch_size,
                              validation_data = test_generator,
                              validation_steps = len(data_test)//batch_size,
                              callbacks=[lr_plateau, early_stopping, checkpoint])

### Set the path and name you want !
##
#
save_name = "/content/gdrive/MyDrive/Colab Notebooks/modele_genus/model_label0_effnetB1_fin_20210627.h5"
save_name2 = "/content/gdrive/MyDrive/Colab Notebooks/modele_genus/model_label0_effnetB1_fin_weights_20210627.h5"
#
##
###

model.save(save_name)
model.save_weights(save_name2)<jupyter_output><empty_output><jupyter_text>**Modèle famille 2 --> label1**<jupyter_code>df = pd.read_csv(csv_label1)
df['filepath'] = [folderpath + str(df['image_id'][x]) + ".jpg" for x in range(len(df))]

display(df.head())

data_train, data_test = train_test_split(df, test_size=0.2, random_state=42)
data_train.head()

batch_size = 32

data_train["label"] = data_train["label"].astype(str)
data_test["label"] = data_test["label"].astype(str)

train_generator = train_data_generator.flow_from_dataframe(dataframe = data_train, directory="", x_col = "filepath", y_col="label", target_size=(256,256), batch_size=batch_size, class_mode='sparse')
test_generator = test_data_generator.flow_from_dataframe(dataframe = data_test, directory="", x_col = "filepath", y_col="label", target_size=(256,256), batch_size=batch_size, class_mode='sparse')

num_class = df["genus"].nunique()

efficientNet = EfficientNetB1(include_top=False, input_shape=(256, 256, 3))
for layer in efficientNet.layers[-1:]:
    layer.trainable = True

model = Sequential()
model.add(efficientNet)
model.add(GlobalAveragePooling2D())
model.add(Dense(units = 1024, activation = 'relu'))
model.add(Dropout(0.2))
model.add(Dense(units = 512, activation = 'relu'))
model.add(Dropout(0.2))
model.add(Dense(units = num_class, activation = 'softmax'))

model.compile(optimizer = 'Adam', loss = 'sparse_categorical_crossentropy', metrics = ['accuracy'])

lr_plateau = callbacks.ReduceLROnPlateau(monitor = 'val_loss',
                                         patience = 3,
                                         factor = 0.1,
                                         verbose = 2,
                                         mode = 'min')

early_stopping = callbacks.EarlyStopping(monitor = "val_loss",
                                         patience = 7,
                                         mode = 'min',
                                         verbose = 2,
                                         restore_best_weights= True)

checkpoint = callbacks.ModelCheckpoint(filepath="/content/gdrive/MyDrive/Colab Notebooks/modele_genus/checkpoint_genus_fam2/model_label1_effnetB1_temp_20210627.h5",
                                      monitor = 'val_loss',
                                      save_best_only = True,
                                      save_weights_only = False,
                                      mode = 'min',
                                      save_freq = 'epoch')


history = model.fit(train_generator, 
                              epochs = 40,
                              steps_per_epoch = len(data_train)//batch_size,
                              validation_data = test_generator,
                              validation_steps = len(data_test)//batch_size,
                              callbacks=[lr_plateau, early_stopping, checkpoint])

### Set the path and name you want !
##
#
save_name = "/content/gdrive/MyDrive/Colab Notebooks/modele_genus/model_label1_effnetB1_fin_20210627.h5"
save_name2 = "/content/gdrive/MyDrive/Colab Notebooks/modele_genus/model_label1_effnetB1_fin_weights_20210627.h5"
#
##
###

model.save(save_name)
model.save_weights(save_name2)<jupyter_output><empty_output><jupyter_text>**Modèle famille 3 --> label2**<jupyter_code>df = pd.read_csv(csv_label2)
df['filepath'] = [folderpath + str(df['image_id'][x]) + ".jpg" for x in range(len(df))]
display(df.head())

data_train, data_test = train_test_split(df, test_size=0.2, random_state=42)
#data_train.head()

batch_size = 32

data_train["label"] = data_train["label"].astype(str)
data_test["label"] = data_test["label"].astype(str)

train_generator = train_data_generator.flow_from_dataframe(dataframe = data_train, directory="", x_col = "filepath", y_col="label", target_size=(256,256), batch_size=batch_size, class_mode='sparse')
test_generator = test_data_generator.flow_from_dataframe(dataframe = data_test, directory="", x_col = "filepath", y_col="label", target_size=(256,256), batch_size=batch_size, class_mode='sparse')

num_class = df["genus"].nunique()

efficientNet = EfficientNetB1(include_top=False, input_shape=(256, 256, 3))
for layer in efficientNet.layers[-1:]:
    layer.trainable = True

model = Sequential()
model.add(efficientNet)
model.add(GlobalAveragePooling2D())
model.add(Dense(units = 1024, activation = 'relu'))
model.add(Dropout(0.2))
model.add(Dense(units = 512, activation = 'relu'))
model.add(Dropout(0.2))
model.add(Dense(units = num_class, activation = 'softmax'))

model.compile(optimizer = 'Adam', loss = 'sparse_categorical_crossentropy', metrics = ['accuracy'])

lr_plateau = callbacks.ReduceLROnPlateau(monitor = 'val_loss',
                                         patience = 3,
                                         factor = 0.1,
                                         verbose = 2,
                                         mode = 'min')

early_stopping = callbacks.EarlyStopping(monitor = "val_loss",
                                         patience = 7,
                                         mode = 'min',
                                         verbose = 2,
                                         restore_best_weights= True)

checkpoint = callbacks.ModelCheckpoint(filepath="/content/gdrive/MyDrive/Colab Notebooks/modele_genus/checkpoint_genus_fam3/model_label2_effnetB1_temp_20210627.h5",
                                      monitor = 'val_loss',
                                      save_best_only = True,
                                      save_weights_only = False,
                                      mode = 'min',
                                      save_freq = 'epoch')


history = model.fit(train_generator, 
                              epochs = 40,
                              steps_per_epoch = len(data_train)//batch_size,
                              validation_data = test_generator,
                              validation_steps = len(data_test)//batch_size,
                              callbacks=[lr_plateau, early_stopping, checkpoint])

### Set the path and name you want !
##
#
save_name = "/content/gdrive/MyDrive/Colab Notebooks/modele_genus/model_label2_effnetB1_fin_20210627.h5"
save_name2 = "/content/gdrive/MyDrive/Colab Notebooks/modele_genus/model_label2_effnetB1_fin_weights_20210627.h5"
#
##
###

model.save(save_name)
model.save_weights(save_name2)<jupyter_output><empty_output><jupyter_text>**Modèle famille 4 --> label3**<jupyter_code>df = pd.read_csv(csv_label3)

df['filepath'] = [folderpath + str(df['image_id'][x]) + ".jpg" for x in range(len(df))]

display(df.head())

data_train, data_test = train_test_split(df, test_size=0.2, random_state=42)
data_train.head()

batch_size = 32

data_train["label"] = data_train["label"].astype(str)
data_test["label"] = data_test["label"].astype(str)

train_generator = train_data_generator.flow_from_dataframe(dataframe = data_train, directory="", x_col = "filepath", y_col="label", target_size=(256,256), batch_size=batch_size, class_mode='sparse')
test_generator = test_data_generator.flow_from_dataframe(dataframe = data_test, directory="", x_col = "filepath", y_col="label", target_size=(256,256), batch_size=batch_size, class_mode='sparse')

num_class = df["genus"].nunique()

efficientNet = EfficientNetB1(include_top=False, input_shape=(256, 256, 3))
for layer in efficientNet.layers[-1:]:
    layer.trainable = True

model = Sequential()
model.add(efficientNet)
model.add(GlobalAveragePooling2D())
model.add(Dense(units = 1024, activation = 'relu'))
model.add(Dropout(0.2))
model.add(Dense(units = 512, activation = 'relu'))
model.add(Dropout(0.2))
model.add(Dense(units = num_class, activation = 'softmax'))

model.compile(optimizer = 'Adam', loss = 'sparse_categorical_crossentropy', metrics = ['accuracy'])

lr_plateau = callbacks.ReduceLROnPlateau(monitor = 'val_loss',
                                         patience = 3,
                                         factor = 0.1,
                                         verbose = 2,
                                         mode = 'min')

early_stopping = callbacks.EarlyStopping(monitor = "val_loss",
                                         patience = 7,
                                         mode = 'min',
                                         verbose = 2,
                                         restore_best_weights= True)

checkpoint = callbacks.ModelCheckpoint(filepath="/content/gdrive/MyDrive/Colab Notebooks/modele_genus/checkpoint_genus_fam4/model_label3_effnetB1_temp_20210627.h5",
                                      monitor = 'val_loss',
                                      save_best_only = True,
                                      save_weights_only = False,
                                      mode = 'min',
                                      save_freq = 'epoch')


history = model.fit(train_generator, 
                              epochs = 40,
                              steps_per_epoch = len(data_train)//batch_size,
                              validation_data = test_generator,
                              validation_steps = len(data_test)//batch_size,
                              callbacks=[lr_plateau, early_stopping, checkpoint])

### Set the path and name you want !
##
#
save_name = "/content/gdrive/MyDrive/Colab Notebooks/modele_genus/model_label3_effnetB1_fin_20210627.h5"
save_name2 = "/content/gdrive/MyDrive/Colab Notebooks/modele_genus/model_label3_effnetB1_fin_weights_20210627.h5"
#
##
###

model.save(save_name)
model.save_weights(save_name2)<jupyter_output><empty_output><jupyter_text>**Modèle famille 5 --> label4**<jupyter_code>df = pd.read_csv(csv_label4)
df['filepath'] = [folderpath + str(df['image_id'][x]) + ".jpg" for x in range(len(df))]

display(df.head())

data_train, data_test = train_test_split(df, test_size=0.2, random_state=42)
data_train.head()

batch_size = 32

data_train["label"] = data_train["label"].astype(str)
data_test["label"] = data_test["label"].astype(str)

train_generator = train_data_generator.flow_from_dataframe(dataframe = data_train, directory="", x_col = "filepath", y_col="label", target_size=(256,256), batch_size=batch_size, class_mode='sparse')
test_generator = test_data_generator.flow_from_dataframe(dataframe = data_test, directory="", x_col = "filepath", y_col="label", target_size=(256,256), batch_size=batch_size, class_mode='sparse')

num_class = df["genus"].nunique()

efficientNet = EfficientNetB1(include_top=False, input_shape=(256, 256, 3))
for layer in efficientNet.layers[-1:]:
    layer.trainable = True

model = Sequential()
model.add(efficientNet)
model.add(GlobalAveragePooling2D())
model.add(Dense(units = 1024, activation = 'relu'))
model.add(Dropout(0.2))
model.add(Dense(units = 512, activation = 'relu'))
model.add(Dropout(0.2))
model.add(Dense(units = num_class, activation = 'softmax'))

model.compile(optimizer = 'Adam', loss = 'sparse_categorical_crossentropy', metrics = ['accuracy'])

lr_plateau = callbacks.ReduceLROnPlateau(monitor = 'val_loss',
                                         patience = 3,
                                         factor = 0.1,
                                         verbose = 2,
                                         mode = 'min')

early_stopping = callbacks.EarlyStopping(monitor = "val_loss",
                                         patience = 7,
                                         mode = 'min',
                                         verbose = 2,
                                         restore_best_weights= True)

checkpoint = callbacks.ModelCheckpoint(filepath="/content/gdrive/MyDrive/Colab Notebooks/modele_genus/checkpoint_genus_fam5/model_label4_effnetB1_temp_20210627.h5",
                                      monitor = 'val_loss',
                                      save_best_only = True,
                                      save_weights_only = False,
                                      mode = 'min',
                                      save_freq = 'epoch')


history = model.fit(train_generator, 
                              epochs = 40,
                              steps_per_epoch = len(data_train)//batch_size,
                              validation_data = test_generator,
                              validation_steps = len(data_test)//batch_size,
                              callbacks=[lr_plateau, early_stopping, checkpoint])

### Set the path and name you want !
##
#
save_name = "/content/gdrive/MyDrive/Colab Notebooks/modele_genus/model_label4_effnetB1_fin_20210627.h5"
save_name2 = "/content/gdrive/MyDrive/Colab Notebooks/modele_genus/model_label4_effnetB1_fin_weights_20210627.h5"
#
##
###

model.save(save_name)
model.save_weights(save_name2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11578521004513366, 'fraction_numerical': 0.026212243953246152, 'mean_word_length': 2.380868544600939, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 120, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '631815', 'license': 'MIT', 'n_tokens_mistral': 5523, 'n_tokens_neox': 5174, 'n_words': 796, 'path': 'models/iteration_2/Modeles_predict genus_20210627.ipynb', 'repo_name': 'amorea04/MushPy', 'size': '17282'}