<jupyter_start><jupyter_text>This notebook contains experiments with two learning schedulers.1. Step Decay2. Exponential DecayThe experiment is observe how these learning schedules impact learning loss and out of set performance.It has been found by learning rate finder starting learning rate from 1e-2 to 0.075e-4<jupyter_code>import os
import math
import shutil
import random
import time
import datetime

import numpy as np
import tensorflow as tf
from tensorflow.keras import layers,models,utils,optimizers
from tensorflow.keras.layers import Dense,Flatten
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.callbacks import TensorBoard,LearningRateScheduler

from util.custom_callbacks import SkipLoggerCallback,LearningRateHistoryCallback
from util.custom_callbacks import StepDecayHistoryCallback, step_decay, exp_decay, ExpDecayHistoryCallback

import matplotlib.pyplot as plt

if tf.test.gpu_device_name():
    print('Default GPU Device: {}'.format(tf.test.gpu_device_name()))
else:
    print("Please install GPU version of TF")
DATA_DIR = 'data/caps_and_shoes_squared/'
IMAGE_SIZE = (28,28)
CLASS_LIST = ['cap','shoes']

train_path = DATA_DIR + 'train'
validation_path = DATA_DIR + 'val'
test_path = DATA_DIR + 'test'

SOURCE_DATA_DIR = 'data/caps_and_shoes_squared_2/'
TEST_DATA_DIR = 'data/out_of_set_tests/'
def create_test_set(seed=random.randint(1,10000)):

    dirpath = os.path.join(TEST_DATA_DIR, 'temp_test')
    if os.path.exists(dirpath) and os.path.isdir(dirpath):
        shutil.rmtree(dirpath)
        
    os.makedirs(dirpath, exist_ok=True)
    
    for class_name in CLASS_LIST:
        src_dir = os.path.join(SOURCE_DATA_DIR, class_name)
        dest_dir = os.path.join(dirpath, class_name)
        os.makedirs(dest_dir, exist_ok=True)
        
        src_files = (os.listdir(src_dir))
        random.seed(seed)
        choices = random.sample(src_files, 25)
        for file_name in choices:
            shutil.copy(os.path.join(src_dir, file_name), dest_dir)
            
def get_oos_test_accuracy(model=None):
    
    out_test_accuracy_log = []
    
    for iter_count in range(25):
    
        seed = random.randint(1,10000)
        create_test_set(seed)

        test_img_generator = ImageDataGenerator(rescale = 1./255)
        test_batches = test_img_generator.flow_from_directory(os.path.join(TEST_DATA_DIR, 'temp_test'),
                                                              target_size=IMAGE_SIZE, classes=CLASS_LIST, color_mode = 'grayscale', 
                                                              class_mode = 'categorical', batch_size=32)
        test_loss, test_accuracy = model.evaluate_generator(test_batches, workers=4, verbose=0) 
        out_test_accuracy_log.append(test_accuracy)
    
    return (np.mean(out_test_accuracy_log),np.std(out_test_accuracy_log))
def define_model_architecture_64_32_16(sgd_optimizer = 'sgd'):
    model = models.Sequential()
    model.add(Flatten(input_shape = (28,28,1)))
    model.add(Dense(64, activation='relu'))
    model.add(Dense(32, activation='relu'))
    model.add(Dense(16, activation='relu'))
    model.add(Dense(2, activation='softmax'))
    
    model.compile(optimizer=sgd_optimizer, loss='binary_crossentropy', metrics=['accuracy'])
    return model
def get_generators_for_batch_size(batch_size_p = 32):
    test_img_generator = ImageDataGenerator( rescale = 1./255)
    train_validation_img_generator = ImageDataGenerator(validation_split=0.2, rescale = 1./255, rotation_range = 10, width_shift_range = 0.1, height_shift_range=0.1, zoom_range = 0.1, 
                                     horizontal_flip=True, fill_mode='nearest')

    train_batches = train_validation_img_generator.flow_from_directory(train_path, target_size=IMAGE_SIZE, classes=CLASS_LIST, color_mode = 'grayscale', 
                                                      class_mode = 'categorical', batch_size=batch_size_p, subset='training')

    validation_batches = train_validation_img_generator.flow_from_directory(train_path, target_size=IMAGE_SIZE, classes=CLASS_LIST, color_mode = 'grayscale', 
                                                      class_mode = 'categorical', batch_size=batch_size_p, subset='validation')

    test_batches = test_img_generator.flow_from_directory(test_path, target_size=IMAGE_SIZE, classes=CLASS_LIST, color_mode = 'grayscale', 
                                                      class_mode = 'categorical',batch_size=batch_size_p, shuffle=False)
    
    return train_batches,validation_batches,test_batches<jupyter_output><empty_output><jupyter_text>Step Decay<jupyter_code>BATCH_SIZES = [32,64]
learning_history_dict = {}
training_time_dict = {}
out_of_set_test_dict = {}
lr_dict = {}
EPOCHS = 600
!rm -rf ./logs/
skipLogger_callback = SkipLoggerCallback(n=25)

lrate_callback = LearningRateScheduler(step_decay)
lr_history_callback = StepDecayHistoryCallback()

sgd =  optimizers.SGD(lr=0.0)
for batch_size in BATCH_SIZES:
    print('working with batch size : %f' % batch_size)
    print('')
    
    log_dir="logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
    tb_callback = TensorBoard(log_dir=log_dir, histogram_freq=1)

    callback_list = [lrate_callback, tb_callback, skipLogger_callback, lr_history_callback]
    
    train_batches,validation_batches,test_batches = get_generators_for_batch_size(batch_size)
    
    model = define_model_architecture_64_32_16(sgd_optimizer=sgd)
    
    start = time.time()
    history = model.fit_generator(generator = train_batches, steps_per_epoch=train_batches.samples/batch_size, 
                                  validation_data = validation_batches, validation_steps = validation_batches.samples/batch_size, 
                                  epochs=EPOCHS, verbose=0, workers=4, callbacks = callback_list)
    
    training_time_dict[str(batch_size)] = time.time() - start
    learning_history_dict[str(batch_size)] = history
    out_of_set_test_dict[str(batch_size)] = get_oos_test_accuracy(model)
    lr_dict[str(batch_size)] = lr_history_callback.lr
    
    test_loss, test_accuracy = model.evaluate_generator(test_batches, workers=4, verbose=0) 
    print('Test loss: %.4f accuracy: %.4f' % (test_loss, test_accuracy))
    
    print('-'*120)
plt.figure(num=None, figsize=(16, 6))

for lr,training_history in learning_history_dict.items():
    plt.plot(training_history.history['loss'], label= lr +'.train')
    plt.plot(training_history.history['val_loss'], label= lr + '.validation')

plt.legend()
plt.title('Training and validation loss vs batch size')
plt.xlim(0, EPOCHS)
plt.show()
plt.figure(figsize=(16, 6))
plt.plot(lr_dict['32'], label='learning rate')
plt.xlabel("epochs")
plt.xlim(0, EPOCHS)
plt.ylabel("learning rate")
plt.legend(loc=0)
plt.grid(True)
plt.title("Learning rate")
plt.show()
training_time_dict
out_of_set_test_dict<jupyter_output><empty_output><jupyter_text>Exponential Decay<jupyter_code>BATCH_SIZES = [32,64]
learning_history_dict = {}
training_time_dict = {}
out_of_set_test_dict = {}
lr_dict = {}
EPOCHS = 600
!rm -rf ./logs/
skipLogger_callback = SkipLoggerCallback(n=25)

lrate_callback = LearningRateScheduler(exp_decay)
lr_history_callback = ExpDecayHistoryCallback()

sgd =  optimizers.SGD(lr=0.0)
for batch_size in BATCH_SIZES:
    print('working with batch size : %f' % batch_size)
    print('')
    
    log_dir="logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
    tb_callback = TensorBoard(log_dir=log_dir, histogram_freq=1)

    callback_list = [lrate_callback, tb_callback, skipLogger_callback, lr_history_callback]
    
    train_batches,validation_batches,test_batches = get_generators_for_batch_size(batch_size)
    
    model = define_model_architecture_64_32_16(sgd_optimizer=sgd)
    
    start = time.time()
    history = model.fit_generator(generator = train_batches, steps_per_epoch=train_batches.samples/batch_size, 
                                  validation_data = validation_batches, validation_steps = validation_batches.samples/batch_size, 
                                  epochs=EPOCHS, verbose=0, workers=4, callbacks = callback_list)
    
    training_time_dict[str(batch_size)] = time.time() - start
    learning_history_dict[str(batch_size)] = history
    out_of_set_test_dict[str(batch_size)] = get_oos_test_accuracy(model)
    lr_dict[str(batch_size)] = lr_history_callback.lr
    
    test_loss, test_accuracy = model.evaluate_generator(test_batches, workers=4, verbose=0) 
    print('Test loss: %.4f accuracy: %.4f' % (test_loss, test_accuracy))
    
    print('-'*120)
plt.figure(num=None, figsize=(16, 6))

for lr,training_history in learning_history_dict.items():
    plt.plot(training_history.history['loss'], label= lr +'.train')
    plt.plot(training_history.history['val_loss'], label= lr + '.validation')

plt.legend()
plt.title('Training and validation loss vs batch size')
plt.xlim(0, EPOCHS)
plt.show()
plt.figure(figsize=(16, 6))
plt.plot(lr_dict['32'], label='learning rate')
plt.xlabel("epochs")
plt.xlim(0, EPOCHS)
plt.ylabel("learning rate")
plt.legend(loc=0)
plt.grid(True)
plt.title("Learning rate")
plt.show()
training_time_dict
out_of_set_test_dict<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10753387533875339, 'fraction_numerical': 0.016476964769647696, 'mean_word_length': 4.17152466367713, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '567955', 'license': 'Apache-2.0', 'n_tokens_mistral': 3201, 'n_tokens_neox': 3040, 'n_words': 510, 'path': '4_3_Tuning_learning_rate_2.ipynb', 'repo_name': 'yantraguru/deeplearn', 'size': '9225'}