<jupyter_start><jupyter_text>Initial setup<jupyter_code>!git clone https://github.com/sayakpaul/Adaptive-Gradient-Clipping
import sys
sys.path.append("Adaptive-Gradient-Clipping")
import tensorflow as tf
tf.random.set_seed(42)
print(tf.__version__)
try: # detect TPUs
    tpu = None
    tpu = tf.distribute.cluster_resolver.TPUClusterResolver() # TPU detection
    tf.config.experimental_connect_to_cluster(tpu)
    tf.tpu.experimental.initialize_tpu_system(tpu)
    strategy = tf.distribute.experimental.TPUStrategy(tpu)
except ValueError: # detect GPUs
    strategy = tf.distribute.MirroredStrategy() # for GPU or multi-GPU machines
  
print("Number of accelerators: ", strategy.num_replicas_in_sync)
import agc
import dataloader
import models
import time
import utils<jupyter_output><empty_output><jupyter_text>Load flowers dataset<jupyter_code>BATCH_SIZE = 64 * strategy.num_replicas_in_sync
train_filenames, valid_filenames, train_steps = dataloader.create_steps(batch_size=BATCH_SIZE, validation_split=0.15)
print(f"Total training samples: {dataloader.count_data_items(train_filenames)}")
print(f"Total validation samples: {dataloader.count_data_items(valid_filenames)}")
print(f"Global batch size: {BATCH_SIZE}")
train_ds = dataloader.get_training_dataset(batch_size=BATCH_SIZE, filenames=train_filenames)
validation_ds = dataloader.get_validation_dataset(batch_size=BATCH_SIZE, filenames=valid_filenames)
samples_images, sample_labels = next(iter(train_ds))
print(samples_images.shape, sample_labels.shape)<jupyter_output>(512, 96, 96, 3) (512, 5)<jupyter_text>Compile model and define training callbacks<jupyter_code>with strategy.scope():
    model = models.get_mini_vgg(use_bn=True)

model.compile(
    optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"]
)
print(f"Total learnable parameters: {model.count_params()/1e6} M")
train_callbacks = [
    tf.keras.callbacks.EarlyStopping(
        monitor="val_loss", patience=10,
        restore_best_weights=True
    ),
    tf.keras.callbacks.ReduceLROnPlateau(
        monitor="val_loss", factor=0.5,
        patience=3, verbose=1
    )
]<jupyter_output><empty_output><jupyter_text>Model training and evaluation<jupyter_code>start = time.time()
history = model.fit(train_ds,
                   steps_per_epoch=train_steps,
                   validation_data=validation_ds,
                   callbacks=train_callbacks,
                   epochs=100)
print(f"Total training time: {(time.time() - start)/60.} minutes")
utils.plot_history(history)
_, accuracy = model.evaluate(validation_ds)
print("Validation accuracy: {:.2f}%".format(accuracy*100))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10931021485111195, 'fraction_numerical': 0.013569543912551827, 'mean_word_length': 5.911458333333333, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '654657', 'license': 'MIT', 'n_tokens_mistral': 932, 'n_tokens_neox': 894, 'n_words': 138, 'path': 'BatchNorm.ipynb', 'repo_name': 'sayakpaul/Adaptive-Gradient-Clipping', 'size': '2653'}