<jupyter_start><jupyter_text>Using tf.data API for increasing the speed of training<jupyter_code>train_ds=tf.data.Dataset.from_tensor_slices((train_img_batch,train_label_batch))
test_ds=tf.data.Dataset.from_tensor_slices((test_img_batch,test_label_batch))
train_ds = train_ds.shuffle(1024).repeat().batch(BATCH_SIZE).cache()
test_ds = test_ds.repeat().batch(BATCH_SIZE).cache()<jupyter_output><empty_output><jupyter_text>Model Creation<jupyter_code>do_fine_tuning = True #@param {type:"boolean"}
def create_model():
  print("Building model with", MODULE_HANDLE)
  model = tf.keras.Sequential([
      # Explicitly define the input shape so the model can be properly
      # loaded by the TFLiteConverter
      tf.keras.layers.InputLayer(input_shape=IMAGE_SIZE + (3,)),
      hub.KerasLayer(MODULE_HANDLE, trainable=do_fine_tuning),
      tf.keras.layers.Dropout(rate=0.2),
      tf.keras.layers.Dense(train_ds.num_classes,
                            kernel_regularizer=tf.keras.regularizers.l2(0.0001))
  ])
  model.build((None,)+IMAGE_SIZE+(3,))
  model.summary()
  model.compile(optimizer=tf.keras.optimizers.SGD(lr=0.005, momentum=0.9), 
  loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True, label_smoothing=0.1),
  metrics=['accuracy'])
  return model
    # model = Sequential([
    #     Conv2D(16,(3,3),activation='relu',input_shape=(IMG_HEIGHT,IMG_WIDTH,3)),
    #     MaxPool2D(),
    #     Dropout(0.5),
    #     Conv2D(32,(3,3),activation='relu'),
    #     MaxPool2D(),
    #     Dropout(0.4),
    #     Flatten(),
    #     Dense(128,activation='relu'),
    #     Dense(5, activation='softmax')
    # ])
    # model.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['accuracy'])
   
def plot_confusion_matrix(cm, class_names):
  """
  Returns a matplotlib figure containing the plotted confusion matrix.

  Args:
    cm (array, shape = [n, n]): a confusion matrix of integer classes
    class_names (array, shape = [n]): String names of the integer classes
  """
  figure = plt.figure(figsize=(8, 8))
  plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Blues)
  plt.title("Confusion matrix")
  plt.colorbar()
  tick_marks = np.arange(len(class_names))
  plt.xticks(tick_marks, class_names, rotation=45)
  plt.yticks(tick_marks, class_names)

  # Compute the labels from the normalized confusion matrix.
  labels = np.around(cm.astype('float') / cm.sum(axis=1)[:, np.newaxis], decimals=2)

  # Use white text if squares are dark; otherwise black.
  threshold = cm.max() / 2.
  for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
    color = "white" if cm[i, j] > threshold else "black"
    plt.text(j, i, labels[i, j], horizontalalignment="center", color=color)

  plt.tight_layout()
  plt.ylabel('True label')
  plt.xlabel('Predicted label')
  return figure
# from datetime import datetime
# from tensorflow import keras
# # Clear out prior logging data.
# !rm -rf /content/gdrive/MyDrive/MyCNN/logs/image

# logdir = "/content/drive/MyDrive/MyCNN/logs/image/" + datetime.now().strftime("%Y%m%d-%H%M%S")
# # Define the basic TensorBoard callback.
# tensorboard_callback = keras.callbacks.TensorBoard(log_dir=logdir)
# file_writer_cm = tf.summary.create_file_writer(logdir + '/cm')
# from sklearn.metrics import classification_report
# from sklearn.metrics import confusion_matrix
# def log_confusion_matrix(epoch, logs):
#   # Use the model to predict the values from the validation dataset.
#   # test_pred_raw = model.predict(x1)
#   # test_pred = np.argmax(test_pred_raw, axis=1)
#   Y_pred = model.predict_generator(valid_data_gen, valid_data_gen.samples // BATCH_SIZE+1)
#   y_pred = np.argmax(Y_pred, axis=1)
#   # Calculate the confusion matrix.
#   # cm = sklearn.metrics.confusion_matrix(valid_labels, test_pred)
#   cm = confusion_matrix(valid_data_gen.classes, y_pred)
#   # Log the confusion matrix as an image summary.
#   figure = plot_confusion_matrix(cm, class_names=list(train_generator.class_indices.keys()))
#   cm_image = plot_to_image(figure)

#   # Log the confusion matrix as an image summary.
#   with file_writer_cm.as_default():
#     tf.summary.image("Confusion Matrix", cm_image, step=epoch)

# # Define the per-epoch callback.
# cm_callback = keras.callbacks.LambdaCallback(on_epoch_end=log_confusion_matrix)
# #Atutomatic rename with epoch number and val accuracy:
# from keras.callbacks import *
# filepath="/content/drive/MyDrive/MyCNN/epochs:{epoch:03d}-val_accuracy:{val_accuracy:.3f}.hdf5"
# checkpoint = ModelCheckpoint(filepath, monitor='val_accuracy', verbose=1, save_best_only=True, mode='max')
# callbacks_list = [tensorboard_callback, cm_callback, checkpoint]
try:
  # %tensorflow_version only exists in Colab.
  %tensorflow_version 2.x
except Exception:
  pass

# Load the TensorBoard notebook extension.
%load_ext tensorboard<jupyter_output><empty_output><jupyter_text>Actually train the model. While the first epoch will be quite a bit slower as we must XLA-compile the execution graph and load the data, later epochs should complete in ~35s.<jupyter_code># Train on TPUs
import time 

# # Start TensorBoard.
# %tensorboard --logdir /content/gdrive/MyDrive/MyCNN/logs/image

with strategy.scope():
    model = create_model()

model.summary()
start = time.time()
model.fit(train_ds,
          steps_per_epoch=STEPS_PER_EPOCH,
          validation_data=test_ds,
          validation_steps=VALIDATION_STEPS,
          # callbacks=callbacks_list,
          epochs=40,
          verbose=2)
print("It took {} seconds".format(time.time() - start))
model.save('Inception-v3-TPU.h5')
model.save("/content/gdrive/MyDrive/MyCNN/Inception-v3-TPU.h5")
plt.plot(model.history.history['accuracy'], label='accuracy')
plt.plot(model.history.history['loss'],label='loss')
plt.plot(model.history.history['val_accuracy'], label = 'val_accuracy')
plt.plot(model.history.history['val_loss'],label='val_loss')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend(loc='upper right')
from matplotlib import pyplot as plt
plt.figure()
plt.ylabel("Loss (training and validation)")
plt.xlabel("Training Steps")
# plt.ylim([0,2])
plt.plot(model.history.history["loss"])
plt.plot(model.history.history["val_loss"])

plt.figure()
plt.ylabel("Accuracy (training and validation)")
plt.xlabel("Training Steps")
# plt.ylim([0,1])
plt.plot(model.history.history["accuracy"])
plt.plot(model.history.history["val_accuracy"])
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.metrics import ConfusionMatrixDisplay

Y_pred = model.predict(valid_data_gen, valid_data_gen.samples // BATCH_SIZE+1)
y_pred = np.argmax(Y_pred, axis=1)
print('Confusion Matrix')
cm = confusion_matrix(valid_data_gen.classes, y_pred)
print(cm)
print('Classification Report')
target_names = list(train_ds.class_indices.keys())
print(classification_report(valid_data_gen.classes, y_pred, target_names=target_names))


disp = ConfusionMatrixDisplay(confusion_matrix=cm,
                              display_labels=valid_data_gen.classes)

# make the plot bigger
fig, ax = plt.subplots(figsize=(10, 10))
disp.plot(cmap=plt.cm.Blues, ax=ax)<jupyter_output><empty_output><jupyter_text>Predictons<jupyter_code># If model is provided, use it to generate predictions.
def display_nine_flowers(images, titles, title_colors=None):
  subplot = 331
  plt.figure(figsize=(13,13))
  for i in range(9):
    color = 'black' if title_colors is None else title_colors[i]
    display_one_flower(images[i], titles[i], 331+i, color)
  plt.tight_layout()
  plt.subplots_adjust(wspace=0.1, hspace=0.1)
  plt.show()
def flower_title(label, prediction):
  # Both prediction (probabilities) and label (one-hot) are arrays with one item per class.
  class_idx = np.argmax(label, axis=-1)
  prediction_idx = np.argmax(prediction, axis=-1)
  if class_idx == prediction_idx:
    return f'{CLASSES[prediction_idx]} [correct]', 'black'
  else:
    return f'{CLASSES[prediction_idx]} [incorrect, should be {CLASSES[class_idx]}]', 'red'

def get_titles(images, labels, model):
  predictions = model.predict(images)
  titles, colors = [], []
  for label, prediction in zip(classes, predictions):
    title, color = flower_title(label, prediction)
    titles.append(title)
    colors.append(color)
  return titles, colors
def get_dataset_iterator(dataset, n_examples):
  return dataset.unbatch().batch(n_examples).as_numpy_iterator()
  
validation_viz_iterator = get_dataset_iterator(valid_data_gen, 9)
# Re-run this cell to show a new batch of images
images, classes = next(validation_viz_iterator)
titles, colors = get_titles(images, classes, model)
display_nine_flowers(images, titles, colors)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12752055874732454, 'fraction_numerical': 0.011152416356877323, 'mean_word_length': 4.615433270082226, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '666606', 'license': 'MIT', 'n_tokens_mistral': 3111, 'n_tokens_neox': 3012, 'n_words': 585, 'path': 'Colab/InceptionV3withTpu.ipynb', 'repo_name': 'Iliescu-Dorin/FaceExpressionEnsemble', 'size': '8877'}