<jupyter_start><jupyter_text>Train Neural Networks<jupyter_code>import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import os
import PIL
import pathlib
from sklearn.utils import class_weight
import pickle

import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from tensorflow.keras.models import Sequential

from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.applications.vgg16 import VGG16
from tensorflow.keras.applications.vgg19 import VGG19
from tensorflow.keras.applications.resnet50 import ResNet50
from tensorflow.keras.applications.inception_v3 import InceptionV3
from tensorflow.keras.applications.xception import Xception
from tensorflow.keras.applications.inception_resnet_v2 import InceptionResNetV2
from tensorflow.keras.applications.densenet import DenseNet121, DenseNet169, DenseNet201

from tensorflow.keras.optimizers import SGD, Adam, RMSprop
from tensorflow.keras.layers import Flatten, Dense, Dropout, GlobalAveragePooling2D, GlobalMaxPooling2D

print(tf.__version__)

print("Num GPUs Available: ", len(tf.config.experimental.list_physical_devices('GPU')))

# %load_ext tensorboard<jupyter_output>2.3.1
Num GPUs Available:  0<jupyter_text>Data  Load data **data is structured as:**             ../data/            dataset/                train/                    Aloe_Vera/                        Aloe_Vera_1.jpeg                        Aloe_Vera_2.jpeg                        ...                    ...                    Umbrella_Tree/                        Umbrella_Tree_1.jpeg                        Umbrella_Tree_2.jpeg                        ...                test/                    Aloe_Vera/                        Aloe_Vera_1.jpeg                        Aloe_Vera_2.jpeg                        ...                    ...                    Umbrella_Tree/                        Umbrella_Tree_1.jpeg                        Umbrella_Tree_2.jpeg                        ...                val/                    Aloe_Vera/                        Aloe_Vera_1.jpeg                        Aloe_Vera_2.jpeg                        ...                    ...                    Umbrella_Tree/                        Umbrella_Tree_1.jpeg                        Umbrella_Tree_2.jpeg                        ...            House_Plants.csv     **Define dataset and parameters**<jupyter_code>data_path = '../data/gsp15_ttv/'
class_names = ['Aloe_Vera', 'Asparagus_Fern', 'Baby_Rubber_Plant', 'Boston_Fern', 'Easter_Lily', 
           'Fiddle_Leaf_Fig', 'Jade_Plant', 'Monstera','Parlor_Palm', 'Peace_Lily', 'Pothos', 
           'Rubber_Plant', 'Snake_Plant', 'Spider_Plant', 'Umbrella_Tree']

img_width, img_height = 224, 224
batch_size = 256<jupyter_output><empty_output><jupyter_text>Load data<jupyter_code>train_data_dir = f'{data_path}/train'
validation_data_dir = f'{data_path}/test'

no_classes = len(class_names)

# import training with augmentation at each epoch
print('Training:')
train_datagen = ImageDataGenerator(
    rescale=1. / 255, 
    shear_range=0.1, 
    zoom_range=0.2, 
    rotation_range=30,
    width_shift_range=0.1,
    height_shift_range=0.1,
    horizontal_flip=True) 
  
train_generator = train_datagen.flow_from_directory( 
    train_data_dir, 
    target_size=(img_width, img_height), 
    batch_size=batch_size, 
    classes=class_names,
    class_mode='categorical',
    seed = 2020,
    shuffle = True) 

# import validation
print('\nValidation:')
val_datagen = ImageDataGenerator(rescale=1. / 255) 
  
validation_generator = val_datagen.flow_from_directory( 
    validation_data_dir, 
    target_size=(img_width, img_height), 
    batch_size=batch_size, 
    classes=class_names,
    class_mode='categorical',
    seed = 2020,
    shuffle = True)<jupyter_output>Training:
Found 10453 images belonging to 15 classes.

Validation:
Found 1321 images belonging to 15 classes.<jupyter_text>Plot data<jupyter_code>plt.figure(figsize=(10, 10))
images, labels = next(train_generator)

for i in range(9):
    ax = plt.subplot(3, 3, i + 1)
    plt.imshow(images[i])
    plt.title(class_names[np.argmax(labels[i])])
    plt.axis("off")<jupyter_output><empty_output><jupyter_text>Define class weights for imbalanced data ![Data Imbalance](../data/figures/number_imgs_per_class.png) Using class_weights changes the range of the loss. This may affect the stability of the training depending on the optimizer. Optimizers whose step size is dependent on the magnitude of the gradient, like optimizers. SGD, may fail. The optimizer used here, optimizers. Adam, is unaffected by the scaling change. Also note that because of the weighting, the total losses are not comparable between the two models.<jupyter_code>def create_weights_dict(train_generator):
    
    '''Calculates the number of samples per class and returns a dictionary for passing to .fit()'''
    
    n_train = len(train_generator.filenames)
    number_of_generator_calls = np.ceil(n_train / (1.0 * batch_size)) 
    # 1.0 above is to skip integer division

    label_list = []
    for i in range(0,int(number_of_generator_calls)):
        label_list.extend(np.array(train_generator[i][1]))
    label_list = list(map(lambda x: np.argmax(x), label_list))

    class_weight_arr = class_weight.compute_class_weight(class_weight='balanced',
                                                 classes = np.unique(label_list),
                                                 y = label_list)

    class_weight_dict = dict(zip(np.arange(len(label_list)), class_weight_arr))
        
    return class_weight_dict
class_weight_dict = create_weights_dict(train_generator)
class_weight_dict<jupyter_output>/opt/miniconda3/envs/tf23/lib/python3.8/site-packages/PIL/TiffImagePlugin.py:785: UserWarning: Corrupt EXIF data.  Expecting to read 4 bytes but only got 0. 
  warnings.warn(str(msg))<jupyter_text>Build model<jupyter_code>def get_prelim():
    model = Sequential([
        layers.experimental.preprocessing.RandomFlip("horizontal_and_vertical", 
                                                     input_shape=(img_height, img_width, 3)),
        layers.experimental.preprocessing.RandomRotation(0.2),
        layers.experimental.preprocessing.RandomZoom(0.3),
        layers.experimental.preprocessing.Rescaling(1./255, input_shape=(img_height, img_width, 3)),
        layers.Conv2D(16, 3, padding='same', activation='relu'),
        layers.MaxPooling2D(),
        layers.Conv2D(32, 3, padding='same', activation='relu'),
        layers.MaxPooling2D(),
        layers.Conv2D(64, 3, padding='same', activation='relu'),
        layers.MaxPooling2D(),
        layers.Flatten(),
        layers.Dense(128, activation='relu'),
        layers.Dense(no_classes),
        layers.Activation('softmax')
    ])
    return model

def get_double_conv():
    model = keras.models.Sequential([
        layers.experimental.preprocessing.RandomFlip("horizontal_and_vertical", 
                                                     input_shape=(img_height, img_width, 3)),
        layers.experimental.preprocessing.RandomRotation(0.2),
        layers.experimental.preprocessing.RandomZoom(0.3),
        layers.experimental.preprocessing.Rescaling(1./255, input_shape=(img_height, img_width, 3)),
        layers.Conv2D(64, 7, activation='relu', padding='same'),
        layers.MaxPooling2D(2),
        layers.Conv2D(128, 3, activation='relu', padding='same'),
        layers.Conv2D(128, 3, activation='relu', padding='same'),
        layers.MaxPooling2D(2),
        layers.Conv2D(256, 3, activation='relu', padding='same'),
        layers.Conv2D(256, 3, activation='relu', padding='same'),
        layers.MaxPooling2D(2),
        layers.Flatten(),
        layers.Dense(128, activation='relu'),
        layers.Dense(64, activation='relu'),
        layers.Dense(no_classes, activation='softmax')
    ])
    return model

def get_VGG16tl():
    resize_layer = layers.experimental.preprocessing.Rescaling(1./255, input_shape=(img_height, img_width, 3))

    # create the base model from the pre-trained model
    pretrained_model = VGG16(input_shape=(img_height, img_width, 3),
                                                         include_top=False,
                                                         weights='imagenet')

    # freeze the convolutional base
    pretrained_model.trainable = False

    model = tf.keras.Sequential([resize_layer,
                                pretrained_model,
                                Flatten(),
                                Dense(256, activation='relu'),
                                Dropout(0.5), # try 0.5
                                Dense(256, activation='relu'),
                                Dense(no_classes, activation='softmax')])
    return model

def get_InceptionV3tl():
    resize_layer = layers.experimental.preprocessing.Rescaling(1./255, input_shape=(img_height, img_width, 3))

    # create the base model from the pre-trained model
    pretrained_model = InceptionV3(input_shape=(img_height, img_width, 3),
                                                         include_top=False,
                                                         weights='imagenet')

    # freeze the convolutional base
    pretrained_model.trainable = False

    model = tf.keras.Sequential([resize_layer,
                                 pretrained_model,
                                 Flatten(),
                                 Dense(256, activation='relu'),
                                 Dropout(0.5),
                                 Dense(256, activation='relu'),
                                 Dense(no_classes, activation='softmax')])
    return model

def get_InceptionV3tl_1024():
    resize_layer = layers.experimental.preprocessing.Rescaling(1./255, input_shape=(img_height, img_width, 3))

    # create the base model from the pre-trained model
    pretrained_model = InceptionV3(input_shape=(img_height, img_width, 3),
                                                         include_top=False,
                                                         weights='imagenet')

    # freeze the convolutional base
    pretrained_model.trainable = False

    model = tf.keras.Sequential([resize_layer,
                                 pretrained_model,
                                 Flatten(),
                                 Dense(1024, activation='relu'),
                                 Dropout(0.2),
                                 Dense(no_classes, activation='softmax')])
    return model


def get_ResNet50tl():
    resize_layer = layers.experimental.preprocessing.Rescaling(1./255, input_shape=(img_height, img_width, 3))

    # create the base model from the pre-trained model
    pretrained_model = ResNet50(input_shape=(img_height, img_width, 3),
                                                         include_top=False,
                                                         weights='imagenet')

    # freeze the convolutional base
    pretrained_model.trainable = False

    model = tf.keras.Sequential([resize_layer,
                                 pretrained_model,
                                 Flatten(),
                                 Dense(256, activation='relu'),
                                 Dropout(0.5),
                                 Dense(256, activation='relu'),
                                 Dense(no_classes, activation='softmax')])
    return model

def get_InceptionResNetV2tl():
    resize_layer = layers.experimental.preprocessing.Rescaling(1./255, input_shape=(img_height, img_width, 3))

    # create the base model from the pre-trained model VGG16
    pretrained_model = InceptionResNetV2(input_shape=(img_height, img_width, 3),
                                                         include_top=False,
                                                         weights='imagenet')

    # freeze the convolutional base
    pretrained_model.trainable = False

    model = tf.keras.Sequential([resize_layer,
                                 pretrained_model,
                                 Flatten(),
                                 Dense(256, activation='relu'),
                                 Dropout(0.5),
                                 Dense(256, activation='relu'),
                                 Dense(no_classes, activation='softmax')])
    return model

def generate_model(model_name):
    
    if model_name == 'prelim':
        model = get_prelim()
        data_augmentation
    
    elif model_name == 'double_conv':
        model = get_double_conv()

    elif model_name == 'VGG16':
        model = get_VGG16tl()
    
    elif model_name == 'ResNet50':
        model = get_ResNet50tl()
    
    elif model_name == 'InceptionV3':
        model = get_InceptionV3tl()
    
    elif model_name == 'InceptionV3_1024':
        model = get_InceptionV3tl_1024()
  
    elif model_name == 'InceptionResNetV2':
        model = get_InceptionResNetV2tl()
    
    else:
        print('please select a valid model')
        
    return model
model = generate_model('InceptionV3_1024')

model.compile(loss = 'categorical_crossentropy',
            optimizer = 'adam',
            metrics = ['accuracy', 'top_k_categorical_accuracy'])

model.summary()<jupyter_output>Model: "sequential_1"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
rescaling_1 (Rescaling)      (None, 224, 224, 3)       0         
_________________________________________________________________
inception_v3 (Functional)    (None, 5, 5, 2048)        21802784  
_________________________________________________________________
flatten_1 (Flatten)          (None, 51200)             0         
_________________________________________________________________
dense_2 (Dense)              (None, 1024)              52429824  
_________________________________________________________________
dropout_1 (Dropout)          (None, 1024)              0         
_________________________________________________________________
dense_3 (Dense)              (None, 15)                15375     
======================================================[...]<jupyter_text>Train model<jupyter_code>initial_epochs=20

history = model.fit(train_generator,
                    steps_per_epoch=len(train_generator.filenames)//batch_size, 
                    epochs=initial_epochs,
                    validation_data=validation_generator,
                    class_weight=class_weight_dict)
# unfreeze the layers
model.trainable = True

model.compile(loss = 'categorical_crossentropy',
            optimizer = keras.optimizers.Adam(1e-5),
            metrics = ['accuracy', 'top_k_categorical_accuracy'])

model.summary()
fine_tune_epochs = 100
total_epochs =  initial_epochs + fine_tune_epochs

history_fine = model.fit(train_generator,
                    steps_per_epoch=len(train_generator.filenames)//batch_size, 
                    epochs=total_epochs,
                    validation_data=validation_generator,
                    class_weight=class_weight_dict)<jupyter_output><empty_output><jupyter_text>Save model/metrics and plot<jupyter_code>model_name = 'InceptionV3_40_200e_GSP1.0'
model.save_weights(f'../models/{model_name}_weights.h5')
model.save(f'../models/{model_name}_model.h5')
acc = history.history['accuracy']
val_acc = history.history['val_accuracy']

t5acc = history.history['top_k_categorical_accuracy']
t5val_acc = history.history['val_top_k_categorical_accuracy']

loss = history.history['loss']
val_loss = history.history['val_loss']

acc += history_fine.history['accuracy']
val_acc += history_fine.history['val_accuracy']

t5acc += history_fine.history['top_k_categorical_accuracy']
t5val_acc += history_fine.history['val_top_k_categorical_accuracy']

loss += history_fine.history['loss']
val_loss += history_fine.history['val_loss']

fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(10,8), sharex=True)

x_plot = np.arange(1, total_epochs+1)

ax[0].plot(x_plot, acc[:total_epochs], '+-', label='training')
ax[0].plot(x_plot, val_acc[:total_epochs], '+-', label='validation')
ax[0].plot(x_plot, t5acc[:total_epochs], '+-', label='top 5 training')
ax[0].plot(x_plot, t5val_acc[:total_epochs], '+-', label='top 5 validation')
ax[0].legend()
ax[0].set_ylabel('accuracy')
# ax[0].set_ylim(0.5, 1)
ax[0].grid(ls='--', c='C7')
ax[0].set_title('accuracy')
ax[0].axvline(initial_epochs, c='C7', ls='--')

ax[1].plot(x_plot, loss[:total_epochs], '+-', label='training')
ax[1].plot(x_plot, val_loss[:total_epochs], '+-', label='validation')
ax[1].legend()
ax[1].set_ylabel('cross entropy')
# ax[1].set_ylim(0, 1)
ax[1].grid(ls='--', c='C7')
ax[1].set_title('loss')
ax[1].set_xlabel('epoch')
ax[1].axvline(initial_epochs, c='C7', ls='--')

plt.show()

plt.savefig(f'../models/{model_name}_graph.svg')
plt.savefig(f'../models/{model_name}_graph.png', dpi=400)
graph_vals = pd.DataFrame({'acc':acc[:total_epochs],
                           'val_acc':val_acc[:total_epochs],
                           'loss':loss[:total_epochs],
                           'val_loss':val_loss[:total_epochs],
                           't5':t5acc[:total_epochs],
                           'val_t5':t5val_acc[:total_epochs]})

graph_vals.to_csv(f'../models/{model_name}_metrics.csv', index=False)
val_predictions = model.predict(val_ds, batch_size=BATCH_SIZE)
def plot_cm(labels, predictions, p=0.5):
    cm = confusion_matrix(labels, predictions > p)
    plt.figure(figsize=(5,5))
    sns.heatmap(cm, annot=True, fmt="d")r4t
    plt.title('Confusion matrix @{:.2f}'.format(p))
    plt.ylabel('Actual label')
    plt.xlabel('Predicted label')
plt.savefig('../models/VGG16_70e_1.0.svg')
model.save_weights('../models/VGG16_20_100e_1.0.h5')
model.save('../models/VGG16_20_100e_1.0.h5')
def plot_confusion_matrix(cm, class_names):
    """
    Returns a matplotlib figure containing the plotted confusion matrix.

    Args:
    cm (array, shape = [n, n]): a confusion matrix of integer classes
    class_names (array, shape = [n]): String names of the integer classes
    """
    figure = plt.figure(figsize=(8, 8))
    plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Blues)
    plt.title("Confusion matrix")
    plt.colorbar()
    tick_marks = np.arange(len(class_names))
    plt.xticks(tick_marks, class_names, rotation=45)
    plt.yticks(tick_marks, class_names)

    # Normalize the confusion matrix.
    cm = np.around(cm.astype('float') / cm.sum(axis=1)[:, np.newaxis], decimals=2)

    # Use white text if squares are dark; otherwise black.
    threshold = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
    color = "white" if cm[i, j] > threshold else "black"
    plt.text(j, i, cm[i, j], horizontalalignment="center", color=color)

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    return figure
# Use the model to predict the values from the validation dataset.
test_pred_raw = model.predict(val_ds)
test_pred = np.argmax(test_pred_raw, axis=1)

# Calculate the confusion matrix.
cm = sklearn.metrics.confusion_matrix(test_labels, test_pred)
# Log the confusion matrix as an image summary.
figure = plot_confusion_matrix(cm, class_names=class_names)
acc = history.history['accuracy']
val_acc = history.history['val_accuracy']

loss = history.history['loss']
val_loss = history.history['val_loss']

fig, ax = plt.subplots(nrows=2, ncols=1, figsize=(10,8), sharex=True)

x_vals = np.arange(1, epochs+1)

ax[0].plot(x_vals, acc, '+-', label='training')
ax[0].plot(x_vals, val_acc, '+-', label='validation')
ax[0].legend()
ax[0].set_ylabel('accuracy')
ax[0].set_ylim(0, 1)
ax[0].grid(ls='--', c='C7')
ax[0].set_title('accuracy')

ax[1].plot(x_vals, loss, '+-', label='training')
ax[1].plot(x_vals, val_loss, '+-', label='validation')
ax[1].legend()
ax[1].set_ylabel('cross entropy')
ax[1].set_ylim(0, 3)
ax[1].grid(ls='--', c='C7')
ax[1].set_title('loss')
ax[1].set_xlabel('epoch')

plt.show()
model.save_weights('../models/.h5')
model.save('../models/.h5')<jupyter_output><empty_output><jupyter_text>Evaluation<jupyter_code>import glob

pred_path = '../data/pred_16c_only1/'

pred_ds = tf.keras.preprocessing.image_dataset_from_directory(
    pred_path,
#     labels = [0]*len(glob.glob(f'{pred_path}*')),
    image_size=(img_height, img_width),
    batch_size=batch_size
    )

normalization_layer = layers.experimental.preprocessing.Rescaling(1./255)
normalized_ds = train_ds.map(lambda x, y: (normalization_layer(x), y))
predictions = model.predict(pred_ds)
print(predictions)
# Generate arg maxes for predictions
classes = np.argmax(predictions, axis = 1)
print(classes[0])
print(class_names[classes[0]])
temp = tf.keras.models.load_model('../models/convmod_1.0.h5')
temp.summary()
dot_img_file = '../models/convmod_1.0.png'
tf.keras.utils.plot_model(model, to_file=dot_img_file, show_shapes=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11976047904191617, 'fraction_numerical': 0.02476000380191997, 'mean_word_length': 2.663474930362117, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 62, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '370477', 'license': 'MIT', 'n_tokens_mistral': 6729, 'n_tokens_neox': 6334, 'n_words': 1137, 'path': 'notebooks/.ipynb_checkpoints/06_train_network-checkpoint.ipynb', 'repo_name': 'Amey-D/PlantPal', 'size': '21042'}