<jupyter_start><jupyter_text>Loading the dataset  How to use tensorflow dataset APIUse list-comprehension to find all image files in the folder<jupyter_code>data_dir=r'C:\Users\kuki\Desktop\Skin\TensorData\train'
data_dir = pathlib.Path(data_dir)
train_image_count = len(list(data_dir.glob('*\*.jpg')))
CLASS_NAMES = np.array([item.name for item in data_dir.glob('*') if item.name != "LICENSE.txt" and item.name != ".DS_store"])
list_ds = tf.data.Dataset.list_files(str(data_dir/'*/*'))
labeled_ds = list_ds.map(process_path, num_parallel_calls=AUTOTUNE)

BATCH_SIZE = 32 
STEPS_PER_EPOCH = train_image_count//BATCH_SIZE
N_VALIDATION=1000
VALIDATION_STEPS = N_VALIDATION // BATCH_SIZE
val_ds= (labeled_ds
               .take(N_VALIDATION)
               .cache("./fibro_val.tfcache")
               .repeat()
               .batch(BATCH_SIZE)
               .prefetch(buffer_size=AUTOTUNE))

shuffle_buffer_size = 100 # affect randomness of dataset. it will take first 100 from dataset and shuffle and pick one.
train_ds = (labeled_ds
            .skip(N_VALIDATION)
            .cache("./fibro_train.tfcache")
            .shuffle(buffer_size=shuffle_buffer_size)
            .repeat()
            .map(augment, num_parallel_calls=AUTOTUNE)
            .batch(BATCH_SIZE)
            .prefetch(buffer_size=AUTOTUNE) #time it takes to produce next element
            )
# no shuffle, no augment
test_data_dir=r'C:\Users\kuki\Desktop\Skin\TensorData\test'
test_data_dir = pathlib.Path(test_data_dir)
test_image_count = len(list(test_data_dir.glob('*\*.jpg')))
TEST_STEPS = test_image_count // BATCH_SIZE
test_list_ds = tf.data.Dataset.list_files(str(test_data_dir/'*/*'))
test_labeled_ds = test_list_ds.map(process_path, num_parallel_calls=AUTOTUNE)
test_ds = (test_labeled_ds
            .cache("./fibro_test.tfcache")
            .repeat()
            .batch(BATCH_SIZE)
            .prefetch(buffer_size=AUTOTUNE) #time it takes to produce next element
            )
TEST_STEPS
#relu, leaky_relu, elu
model_cnnA1= Sequential([
    Conv2D(32, (3, 3), activation='relu', input_shape=(IMG_HEIGHT, IMG_WIDTH ,3)),
    MaxPooling2D((2, 2)),
    Conv2D(64, (3, 3), activation='relu'),
    MaxPooling2D(),
    Conv2D(64, (3, 3), activation='relu'),
    Flatten(),
    Dense(64, activation='relu'),
    Dense(2) #young vs old
])
model_cnnA2= Sequential([
    Conv2D(32, (3, 3), activation='relu', input_shape=(IMG_HEIGHT, IMG_WIDTH ,3)),
    MaxPooling2D((2, 2)),
    Dropout(0.2), 
    Conv2D(64, (3, 3), activation='relu'),
    MaxPooling2D(),
    Conv2D(64, (3, 3), activation='relu'),
    Dropout(0.2),
    Flatten(),
    Dense(64, activation='relu'),
    Dense(2)
])
# Regularization detail on L2, dropout, Early stopping, Data augmentation
# https://www.deeplearningbook.org/contents/regularization.html
model_cnnA3= Sequential([
    Conv2D(32, (3, 3), activation='relu', input_shape=(IMG_HEIGHT, IMG_WIDTH ,3),
           kernel_regularizer=regularizers.l2(0.001)),
    MaxPooling2D((2, 2)),
    Conv2D(64, (3, 3), activation='relu',kernel_regularizer=regularizers.l2(0.001)),
    MaxPooling2D(),
    Conv2D(64, (3, 3), activation='relu',kernel_regularizer=regularizers.l2(0.001)),
    Flatten(),
    Dense(64, activation='relu'),
    Dense(2)
])
model_cnnA4= Sequential([
    Conv2D(32, (3, 3), activation='relu', input_shape=(IMG_HEIGHT, IMG_WIDTH ,3),
           kernel_regularizer=regularizers.l2(0.001)),
    MaxPooling2D((2, 2)),
    Dropout(0.2),
    Conv2D(64, (3, 3), activation='relu',kernel_regularizer=regularizers.l2(0.001)),
    MaxPooling2D(),
    Conv2D(64, (3, 3), activation='relu',kernel_regularizer=regularizers.l2(0.001)),
    Dropout(0.2),
    Flatten(),
    Dense(64, activation='relu'),
    Dense(2)
])

lr_schedule = optimizers.schedules.InverseTimeDecay(
  0.0001,
  decay_steps=STEPS_PER_EPOCH*1000,
  decay_rate=1,
  staircase=False)

checkpoint_path = "training_1/cp.ckpt"
checkpoint_dir = os.path.dirname(checkpoint_path)
shutil.rmtree(checkpoint_dir, ignore_errors=True)

# file_writer = tf.summary.create_file_writer(log_dir/"metrics")
# file_writer.set_as_default()

def get_optimizer():
  return optimizers.Adam(lr_schedule)
class CollectBatchStats(tf.keras.callbacks.Callback):
  def __init__(self):
    self.batch_losses = []
    self.batch_acc = []

  def on_train_batch_end(self, batch, logs=None):
    self.batch_losses.append(logs['loss'])
    self.batch_acc.append(logs['accuracy'])
    self.model.reset_metrics()


def get_callbacks(name):
  return [
    modeling.EpochDots(),
    CollectBatchStats(),
    tf.keras.callbacks.EarlyStopping(monitor='val_sparse_categorical_crossentropy',
                                     patience=100,restore_best_weights=True),
    tf.keras.callbacks.TensorBoard(log_dir/name, histogram_freq=1),
    tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,
                                       verbose=0,
                                       monitor='val_sparse_categorical_crossentropy',
                                       save_weights_only=True,
                                       save_best_only=True),
    # tf.keras.callbacks.ReduceLROnPlateau(monitor='val_sparse_categorical_crossentropy',
    #                                      factor=0.1, patience=10, verbose=0, mode='auto',
    #                                      min_delta=0.0001, cooldown=0, min_lr=0),
  ]
def compilefit(model,loss_fn, name, optimizer=None, max_epochs=10):
    if optimizer is None: optimizer = get_optimizer()
    model.compile(optimizer=optimizer,
              loss=loss_fn,
              metrics=[
                  tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
                  'accuracy'])
    model.summary()
    history = model.fit(train_ds,
          steps_per_epoch= STEPS_PER_EPOCH,
          epochs=max_epochs,
          verbose=0,
          validation_data=val_ds,
          callbacks=get_callbacks(name),
          validation_steps=VALIDATION_STEPS,
          use_multiprocessing=True)
    if not os.path.isdir(os.path.abspath(name)):
        os.mkdir(os.path.abspath(name))
    model.save(pathlib.Path(name)/'full_model.h5')
    return history
%tensorboard --logdir log_dir
histories={}
loss_fn = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
histories['cnnA1'] = compilefit(model_cnnA1,loss_fn,'cnn/A1')
dfobj = histories['cnnA1'].history
dfobj.pop('loss')
dfobj.pop('val_loss')
pd.DataFrame(dfobj).plot()

#loss and accuracy decrease -> overfitting
#loss increase and accuracy decrease -> regularization works only if loss starts to decrease eventually
def compilefit(model,loss_fn, name, optimizer=None, max_epochs=10):
    if optimizer is None: optimizer = get_optimizer()
    model.compile(optimizer=optimizer,
              loss=loss_fn,
              metrics=[
                  tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
                  'accuracy'])
    model.summary()
    history = model.fit(train_ds,
          steps_per_epoch= STEPS_PER_EPOCH,
          epochs=max_epochs,
          verbose=0,
          validation_data=test_ds,
          callbacks=get_callbacks(name),
          validation_steps=TEST_STEPS,
          use_multiprocessing=True)
    if not os.path.isdir(os.path.abspath(name)):
        os.mkdir(os.path.abspath(name))
    model.save(pathlib.Path(name)/'full_model.h5')
    return history
histories={}
histories['cnnA1'] = compilefit(model_cnnA1,loss_fn,'cnn/A1')
histories['cnnA2'] = compilefit(model_cnnA2,loss_fn,'cnn/A2')
histories['cnnA3'] = compilefit(model_cnnA3,loss_fn,'cnn/A3')
histories['cnnA4'] = compilefit(model_cnnA4,loss_fn,'cnn/A4')
def plotdf(dfobj):
    dfobj.pop('loss')
    dfobj.pop('val_loss')
    pd.DataFrame(dfobj).plot()

plotdf(histories['cnnA1'].history)
plotdf(histories['cnnA2'].history)
plotdf(histories['cnnA3'].history)
plotdf(histories['cnnA4'].history)
mirrored_strategy = tf.distribute.MirroredStrategy(devices=None, cross_device_ops=tf.distribute.HierarchicalCopyAllReduce())
with mirrored_strategy.scope():
    model= Sequential([
    Conv2D(32, (3, 3), activation='elu', input_shape=(IMG_HEIGHT, IMG_WIDTH ,3),
           kernel_regularizer=regularizers.l2(0.001)),
    MaxPooling2D((2, 2)),
    Dropout(0.5),
    Conv2D(64, (3, 3), activation='elu',kernel_regularizer=regularizers.l2(0.001)),
    MaxPooling2D(),
    Conv2D(64, (3, 3), activation='elu',kernel_regularizer=regularizers.l2(0.001)),
    Dropout(0.5),
    Flatten(),
    Dense(64, activation='elu'),
    Dense(2)
    ])

    model.compile(optimizer='adam',
              loss=loss_fn,
              metrics=['accuracy'])

history = model.fit(train_ds,
      steps_per_epoch= STEPS_PER_EPOCH,
      epochs=10,
      verbose=0,
      validation_data=test_ds,
      validation_steps=VALIDATION_STEPS,
      use_multiprocessing=True)
pd.DataFrame(history.history).plot()
classifier_url ='https://tfhub.dev/google/efficientnet/b0/classification/1'
feature_extractor_url = "https://tfhub.dev/google/efficientnet/b0/feature-vector/1"
module_spec = hub.load_module_spec(classifier_url)
height, width = hub.get_expected_image_size(module_spec)

# classifier = Sequential([
#     hub.KerasLayer(classifier_url, input_shape=(224,224,3))
# ])

feature_extractor_layer = hub.KerasLayer(feature_extractor_url,
                                         input_shape=(100,100,3))
feature_extractor_layer.trainable = False
model = Sequential([
  feature_extractor_layer,
  Dense(2)
])
model.compile(
  optimizer=optimizers.Adam(),
  loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True),
  metrics=['accuracy'])
batch_stats_callback=CollectBatchStats()
history = model.fit(train_ds, epochs=5,
                              steps_per_epoch=STEPS_PER_EPOCH,
                              callbacks = [batch_stats_callback])
plt.figure()
plt.ylabel("Loss")
plt.xlabel("Training Steps")
plt.ylim([0,1])
plt.plot(batch_stats_callback.batch_losses)
plt.figure()
plt.ylabel("Accuracy")
plt.xlabel("Training Steps")
plt.ylim([0,1])
plt.plot(batch_stats_callback.batch_acc)
pd.DataFrame(history.history).plot()
# Xception 299x299
# MobileNetV2 = 224x224
# InceptionResNetV2 = 299x299
base_model = tf.keras.applications.Xception(input_shape=(100,100,3),
                                            include_top=False,
                                            weights=None,
                                            pooling='avg'
                                            )
base_model.trainable = True
Xception = tf.keras.Sequential([
  base_model,
  Dense(2)
])

base_model.trainable = True
print("Number of layers in the base model: ", len(base_model.layers))
fine_tune_at = 100
for layer in base_model.layers[:fine_tune_at]:
  layer.trainable =  False
model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
              optimizer = tf.keras.optimizers.RMSprop(lr=base_learning_rate/10),
              metrics=['accuracy'])
model.summary()
fine_tune_epochs = 10
initial_epochs = 10
total_epochs =  initial_epochs + fine_tune_epochs
history_fine = model.fit(train_ds,
                         epochs=total_epochs,
                         initial_epoch =  history.epoch[-1],
                         steps_per_epoch= STEPS_PER_EPOCH,
                         validation_data=val_ds,
                         validation_steps=VALIDATION_STEPS,)

                         
acc = history.history['accuracy'] + history_fine.history['accuracy']
val_acc = history.history['val_accuracy'] + history_fine.history['val_accuracy']
loss = history.history['loss'] + history_fine.history['loss']
val_loss = history.history['val_loss'] + history_fine.history['val_loss']
plt.figure(figsize=(8, 8))
plt.subplot(2, 1, 1)
plt.plot(acc, label='Training Accuracy')
plt.plot(val_acc, label='Validation Accuracy')
plt.ylim([0.8, 1])
plt.plot([initial_epochs-1,initial_epochs-1],
          plt.ylim(), label='Start Fine Tuning')
plt.legend(loc='lower right')
plt.title('Training and Validation Accuracy')

plt.subplot(2, 1, 2)
plt.plot(loss, label='Training Loss')
plt.plot(val_loss, label='Validation Loss')
plt.ylim([0, 1.0])
plt.plot([initial_epochs-1,initial_epochs-1],
         plt.ylim(), label='Start Fine Tuning')
plt.legend(loc='upper right')
plt.title('Training and Validation Loss')
plt.xlabel('epoch')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13149995943863066, 'fraction_numerical': 0.02660825829480003, 'mean_word_length': 3.460202604920405, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 19, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '525383', 'license': 'MIT', 'n_tokens_mistral': 4481, 'n_tokens_neox': 4252, 'n_words': 617, 'path': 'test0421.ipynb', 'repo_name': 'kukionfr/Tensorflow-Tutorial-Kyu', 'size': '12327'}