<jupyter_start><jupyter_text>This notebook uses latest ADAM optimizer.<jupyter_code>import os
import shutil
import time
import datetime
import random

import numpy as np

from tensorflow.keras import layers,models,utils,optimizers
from tensorflow.keras.layers import Dense,Flatten
from tensorflow.keras.preprocessing.image import ImageDataGenerator

from tensorflow.keras.callbacks import TensorBoard,EarlyStopping,LearningRateScheduler
from util.custom_callbacks import SkipLoggerCallback,StepDecayHistoryCallback
from util.custom_callbacks import step_decay

from util.learningratefinder import LearningRateFinder

import matplotlib.pyplot as plt
DATA_DIR = 'data/caps_and_shoes_squared/'
IMAGE_SIZE = (28,28)
CLASS_LIST = ['cap','shoes']
BATCH_SIZE = 32

train_path = DATA_DIR + 'train'
validation_path = DATA_DIR + 'val'
test_path = DATA_DIR + 'test'

SOURCE_DATA_DIR = 'data/caps_and_shoes_squared_2/'
TEST_DATA_DIR = 'data/out_of_set_tests/'
def create_test_set(seed=random.randint(1,10000)):

    dirpath = os.path.join(TEST_DATA_DIR, 'temp_test')
    if os.path.exists(dirpath) and os.path.isdir(dirpath):
        shutil.rmtree(dirpath)
        
    os.makedirs(dirpath, exist_ok=True)
    
    for class_name in CLASS_LIST:
        src_dir = os.path.join(SOURCE_DATA_DIR, class_name)
        dest_dir = os.path.join(dirpath, class_name)
        os.makedirs(dest_dir, exist_ok=True)
        
        src_files = (os.listdir(src_dir))
        random.seed(seed)
        choices = random.sample(src_files, 25)
        for file_name in choices:
            shutil.copy(os.path.join(src_dir, file_name), dest_dir)
            
def get_oos_test_accuracy(model=None):
    
    out_test_accuracy_log = []
    
    for iter_count in range(25):
    
        seed = random.randint(1,10000)
        create_test_set(seed)

        test_img_generator = ImageDataGenerator(rescale = 1./255)
        test_batches = test_img_generator.flow_from_directory(os.path.join(TEST_DATA_DIR, 'temp_test'),
                                                              target_size=IMAGE_SIZE, classes=CLASS_LIST, color_mode = 'grayscale', 
                                                              class_mode = 'categorical', batch_size=32)
        test_loss, test_accuracy = model.evaluate_generator(test_batches, workers=4, verbose=0) 
        out_test_accuracy_log.append(test_accuracy)
    
    return (np.mean(out_test_accuracy_log),np.std(out_test_accuracy_log))
test_img_generator = ImageDataGenerator( rescale = 1./255)
train_validation_img_generator = ImageDataGenerator(validation_split=0.2, rescale = 1./255, rotation_range = 10, width_shift_range = 0.1, height_shift_range=0.1, zoom_range = 0.1, 
                                   horizontal_flip=True, fill_mode='nearest')

train_batches = train_validation_img_generator.flow_from_directory(train_path, target_size=IMAGE_SIZE, classes=CLASS_LIST, color_mode = 'grayscale', 
                                                  class_mode = 'categorical', batch_size=BATCH_SIZE, subset='training')

validation_batches = train_validation_img_generator.flow_from_directory(train_path, target_size=IMAGE_SIZE, classes=CLASS_LIST, color_mode = 'grayscale', 
                                                  class_mode = 'categorical', batch_size=BATCH_SIZE, subset='validation')

test_batches = test_img_generator.flow_from_directory(test_path, target_size=IMAGE_SIZE, classes=CLASS_LIST, color_mode = 'grayscale', 
                                                  class_mode = 'categorical',batch_size=1, shuffle=False)
def define_model_architecture_64_32_16(optimizer_p = 'sgd'):
    model = models.Sequential()
    model.add(Flatten(input_shape = (28,28,1)))
    model.add(Dense(64, activation='relu'))
    model.add(Dense(32, activation='relu'))
    model.add(Dense(16, activation='relu'))
    model.add(Dense(2, activation='softmax'))
    
    model.compile(optimizer=optimizer_p, loss='binary_crossentropy', metrics=['accuracy'])
    return model
adam = optimizers.Adam()
model = define_model_architecture_64_32_16(optimizer_p=adam)
lrf = LearningRateFinder(model)
lrf.find(train_batches,1e-10, 1e+1, stepsPerEpoch=train_batches.samples/BATCH_SIZE, batchSize=BATCH_SIZE, verbose=0)
lrf.plot_loss()<jupyter_output><empty_output><jupyter_text>Optimum rate is from 1e-6 to 1e-3<jupyter_code>learning_rates = [1e-6,1e-5,1e-4,1e-3]
learning_history_dict = {}
training_time_dict = {}
out_of_set_test_dict = {}
EPOCHS = 500
!rm -rf ./logs/
skipLogger_callback = SkipLoggerCallback(n=25)
for learning_rate in learning_rates:
    print('working with learning rate : %f' % learning_rate)
    print('')
    
    adam = optimizers.Adam(lr=learning_rate)
    model = define_model_architecture_64_32_16(optimizer_p=adam)
    
    log_dir="logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
    tb_callback = TensorBoard(log_dir=log_dir, histogram_freq=1)
    
    start = time.time()
    history = model.fit_generator(generator = train_batches, steps_per_epoch=train_batches.samples/BATCH_SIZE, 
                                  validation_data = validation_batches, validation_steps = validation_batches.samples/BATCH_SIZE, 
                                  epochs=EPOCHS, verbose=0, workers=4, callbacks = [skipLogger_callback, tb_callback])
    training_time_dict[str(learning_rate)] = time.time() - start
    learning_history_dict[str(learning_rate)] = history
    
    out_of_set_test_dict[str(learning_rate)] = get_oos_test_accuracy(model)
    test_loss, test_accuracy = model.evaluate_generator(test_batches, workers=4, verbose=0) 
    print('Test loss: %.4f accuracy: %.4f' % (test_loss, test_accuracy))    
    print('-'*120)

plt.figure(num=None, figsize=(16, 6))

for lr,training_history in learning_history_dict.items():
    #plt.plot(training_history.history['loss'], label= lr +'.train')
    plt.plot(training_history.history['val_loss'], label= lr + '.validation')

plt.legend()
plt.title('Training and validation accuracy across learning rates')
plt.xlim(0, EPOCHS)
plt.show()<jupyter_output><empty_output><jupyter_text>Finetuning by range finder<jupyter_code>adam = optimizers.Adam()
model = define_model_architecture_64_32_16(optimizer_p=adam)
lrf = LearningRateFinder(model)
lrf.find(train_batches,8e-5, 8e-4, stepsPerEpoch=train_batches.samples/BATCH_SIZE, batchSize=BATCH_SIZE, verbose=0)
lrf.plot_loss()
learning_rates = [7e-5,9.8e-5,1e-4,6e-4]
learning_history_dict = {}
training_time_dict = {}
out_of_set_test_dict = {}
EPOCHS = 300
!rm -rf ./logs/
skipLogger_callback = SkipLoggerCallback(n=25)
for learning_rate in learning_rates:
    print('working with learning rate : %f' % learning_rate)
    print('')
    
    adam = optimizers.Adam(lr=learning_rate)
    model = define_model_architecture_64_32_16(optimizer_p=adam)
    
    log_dir="logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
    tb_callback = TensorBoard(log_dir=log_dir, histogram_freq=1)
    
    start = time.time()
    history = model.fit_generator(generator = train_batches, steps_per_epoch=train_batches.samples/BATCH_SIZE, 
                                  validation_data = validation_batches, validation_steps = validation_batches.samples/BATCH_SIZE, 
                                  epochs=EPOCHS, verbose=0, workers=4, callbacks = [skipLogger_callback, tb_callback])
    training_time_dict[str(learning_rate)] = time.time() - start
    learning_history_dict[str(learning_rate)] = history
    
    out_of_set_test_dict[str(learning_rate)] = get_oos_test_accuracy(model)
    test_loss, test_accuracy = model.evaluate_generator(test_batches, workers=4, verbose=0) 
    print('Test loss: %.4f accuracy: %.4f' % (test_loss, test_accuracy))    
    print('-'*120)

plt.figure(num=None, figsize=(16, 6))

for lr,training_history in learning_history_dict.items():
    #plt.plot(training_history.history['loss'], label= lr +'.train')
    plt.plot(training_history.history['val_loss'], label= lr + '.validation')

plt.legend()
plt.title('Training and validation accuracy across learning rates')
plt.xlim(0, EPOCHS)
plt.show()
plt.figure(num=None, figsize=(16, 6))

plt.plot(learning_history_dict['9.8e-05'].history['val_loss'], label= lr +'.train')
plt.plot(learning_history_dict['7e-05'].history['val_loss'], label= lr + '.validation')

plt.legend()
plt.title('Training and validation accuracy across learning rates')
plt.xlim(0, EPOCHS)
plt.show()
out_of_set_test_dict
training_time_dict<jupyter_output><empty_output><jupyter_text>Learning scheduleStart with learning rate of 7e-4 for 150 epochs then 7e-5 for 250 epochs and then 9e-5 for 200 epochs<jupyter_code>def custom_decay(epoch):
    if epoch < 50 :
        lrate = 1e-3
    elif epoch < 250 :
        lrate = 7e-4
    elif epoch < 450 :
        lrate = 7e-5
    else:
        lrate =  9e-5
        
    return lrate
EPOCHS = 600
!rm -rf ./logs/
skipLogger_callback = SkipLoggerCallback(n=25)

lrate_callback = LearningRateScheduler(custom_decay)

log_dir="logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
tb_callback = TensorBoard(log_dir=log_dir, histogram_freq=1)

callback_list = [skipLogger_callback,lrate_callback,tb_callback]
adam = optimizers.Adam(lr=0.0)
model = define_model_architecture_64_32_16(optimizer_p=adam)
    
log_dir="logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
tb_callback = TensorBoard(log_dir=log_dir, histogram_freq=1)
    
start = time.time()
history = model.fit_generator(generator = train_batches, steps_per_epoch=train_batches.samples/BATCH_SIZE, 
                              validation_data = validation_batches, validation_steps = validation_batches.samples/BATCH_SIZE, 
                              epochs=EPOCHS, verbose=0, workers=4, callbacks = callback_list)

print('time taken : %.4f' % (time.time() - start))
oos_test_loss, oos_test_accuracy = get_oos_test_accuracy(model)
print('OOS Test loss: %.4f OOS accuracy: %.4f' % (oos_test_loss, oos_test_accuracy))    
test_loss, test_accuracy = model.evaluate_generator(test_batches, workers=4, verbose=0) 
print('Test loss: %.4f accuracy: %.4f' % (test_loss, test_accuracy))    

print('-'*120)
plt.figure(figsize=(16, 6))
plt.plot(history.history['loss'], label='train')
plt.plot(history.history['val_loss'], label='validation')
plt.legend()
plt.xlim(0, EPOCHS)
plt.show()
plt.figure(figsize=(16, 6))
plt.plot(history.history['accuracy'], label='train')
plt.plot(history.history['val_accuracy'], label='validation')
plt.legend()
plt.xlim(0, EPOCHS)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1173396674584323, 'fraction_numerical': 0.022707838479809977, 'mean_word_length': 4.268268268268268, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '654703', 'license': 'Apache-2.0', 'n_tokens_mistral': 3769, 'n_tokens_neox': 3544, 'n_words': 580, 'path': '4_4_Adam_Optimizer.ipynb', 'repo_name': 'yantraguru/deeplearn', 'size': '10525'}