<jupyter_start><jupyter_text>Complete example with the best settings found so far<jupyter_code>import os
import shutil
import time
import datetime
import random

import numpy as np
import pandas as pd

from sklearn.metrics import confusion_matrix
from sklearn.metrics import roc_curve, auc

from tensorflow.keras import layers,models,utils,optimizers,regularizers
from tensorflow.keras.layers import Dense,Flatten
from tensorflow.keras.preprocessing.image import ImageDataGenerator

from tensorflow.keras.callbacks import TensorBoard,EarlyStopping,LearningRateScheduler
from util.custom_callbacks import SkipLoggerCallback,StepDecayHistoryCallback,LearningRateHistoryCallback
from util.custom_callbacks import step_decay,step_decay_25_864065
from util.clr_callback import CyclicLR 

import matplotlib.pyplot as plt
import seaborn as sns
DATA_DIR = 'data/caps_and_shoes_squared/'
IMAGE_SIZE = (28,28)
CLASS_LIST = ['cap','shoes']
BATCH_SIZE = 32

train_path = DATA_DIR + 'train'
validation_path = DATA_DIR + 'val'
test_path = DATA_DIR + 'test'

SOURCE_DATA_DIR = 'data/caps_and_shoes_squared_2/'
TEST_DATA_DIR = 'data/out_of_set_tests/'
def create_test_set(seed=random.randint(1,10000)):

    dirpath = os.path.join(TEST_DATA_DIR, 'temp_test')
    if os.path.exists(dirpath) and os.path.isdir(dirpath):
        shutil.rmtree(dirpath)
        
    os.makedirs(dirpath, exist_ok=True)
    
    for class_name in CLASS_LIST:
        src_dir = os.path.join(SOURCE_DATA_DIR, class_name)
        dest_dir = os.path.join(dirpath, class_name)
        os.makedirs(dest_dir, exist_ok=True)
        
        src_files = (os.listdir(src_dir))
        random.seed(seed)
        choices = random.sample(src_files, 25)
        for file_name in choices:
            shutil.copy(os.path.join(src_dir, file_name), dest_dir)
            
def get_oos_test_accuracy(model=None):
    
    out_test_accuracy_log = []
    
    for iter_count in range(50):
    
        seed = random.randint(1,10000)
        create_test_set(seed)

        test_img_generator = ImageDataGenerator(rescale = 1./255)
        test_batches = test_img_generator.flow_from_directory(os.path.join(TEST_DATA_DIR, 'temp_test'),
                                                              target_size=IMAGE_SIZE, classes=CLASS_LIST, color_mode = 'grayscale', 
                                                              class_mode = 'categorical', batch_size=32)
        test_loss, test_accuracy = model.evaluate_generator(test_batches, workers=4, verbose=0) 
        out_test_accuracy_log.append(test_accuracy)
    
    return (np.mean(out_test_accuracy_log),np.std(out_test_accuracy_log))

def plot_images(ims, figsize=(24,12), rows=1, interp=False, titles=None):
    f = plt.figure(figsize=figsize)
    cols = len(ims)//rows if len(ims) % 2 == 0 else len(ims)//rows + 1
    for i in range(len(ims)):
        sp = f.add_subplot(rows, cols, i+1)
        sp.axis('Off')
        if titles is not None:
            sp.set_title(titles[i], fontsize=18)
        
        plt.imshow(ims[i].reshape(IMAGE_SIZE[0],IMAGE_SIZE[1]), cmap="gray")
test_img_generator = ImageDataGenerator( rescale = 1./255)
train_validation_img_generator = ImageDataGenerator(validation_split=0.2, rescale = 1./255, rotation_range = 10, width_shift_range = 0.1, height_shift_range=0.1, zoom_range = 0.1, 
                                   horizontal_flip=True, fill_mode='nearest')

train_batches = train_validation_img_generator.flow_from_directory(train_path, target_size=IMAGE_SIZE, classes=CLASS_LIST, color_mode = 'grayscale', 
                                                  class_mode = 'categorical', batch_size=BATCH_SIZE, subset='training')

validation_batches = train_validation_img_generator.flow_from_directory(train_path, target_size=IMAGE_SIZE, classes=CLASS_LIST, color_mode = 'grayscale', 
                                                  class_mode = 'categorical', batch_size=BATCH_SIZE, subset='validation')

test_batches = test_img_generator.flow_from_directory(test_path, target_size=IMAGE_SIZE, classes=CLASS_LIST, color_mode = 'grayscale', 
                                                  class_mode = 'categorical',batch_size=1, shuffle=False)
def define_model_architecture_64_32_16():
    model = models.Sequential()
    model.add(Flatten(input_shape = (28,28,1)))
    model.add(Dense(64, activation='relu'))
    model.add(Dense(32, activation='relu'))
    model.add(Dense(16, activation='relu'))
    model.add(Dense(2, activation='softmax'))
    return model<jupyter_output><empty_output><jupyter_text>Step Decay Schedule<jupyter_code>EPOCHS = 700

skipLogger_callback = SkipLoggerCallback(n=25)

lrate_callback = LearningRateScheduler(step_decay_25_864065)
callback_list = [lrate_callback, skipLogger_callback]
sgd_optimizer = optimizers.SGD(lr=0.0,momentum=0.9,nesterov=True)
model = define_model_architecture_64_32_16()
model.compile(optimizer=sgd_optimizer, loss='binary_crossentropy', metrics=['accuracy'])
start = time.time()
history_step_decay = model.fit_generator(generator = train_batches, steps_per_epoch=train_batches.samples/BATCH_SIZE, 
                              validation_data = validation_batches, validation_steps = validation_batches.samples/BATCH_SIZE, 
                              epochs=EPOCHS, verbose=0, workers=4, callbacks = callback_list)

print('time taken : %.4f' % (time.time() - start))
oos_test_loss, oos_test_accuracy = get_oos_test_accuracy(model)
print('OOS Test loss: %.4f OOS std dev: %.4f' % (oos_test_loss, oos_test_accuracy))    
test_loss, test_accuracy = model.evaluate_generator(test_batches, workers=4, verbose=0) 
print('Test loss: %.4f accuracy: %.4f' % (test_loss, test_accuracy))    
history_dict = {}
history_dict['step_decay'] = history_step_decay
model.save("model/model_step_decay.h5")
plt.figure(num=None, figsize=(16, 6))

plt.plot(history_step_decay.history['loss'], label= '.train')
plt.plot(history_step_decay.history['val_loss'], label= '.validation')

plt.legend()
plt.title('Training and validation loss vs batch size')
plt.show()
plt.figure(num=None, figsize=(16, 6))

plt.plot(history_step_decay.history['accuracy'], label= '.train')
plt.plot(history_step_decay.history['val_accuracy'], label= '.validation')

plt.legend()
plt.title('Training and validation accuracy vs batch size')
plt.show()<jupyter_output><empty_output><jupyter_text>Triangular Schedule<jupyter_code>MIN_LR = 1e-4
MAX_LR = 2e-3
STEP_SIZE = 8
CLR_METHOD = "triangular"
EPOCHS = 79*6
sgd_optimizer = optimizers.SGD(lr=MIN_LR, momentum=0.9, nesterov=True)
model = define_model_architecture_64_32_16()
model.compile(loss="binary_crossentropy", optimizer=sgd_optimizer, metrics=["accuracy"])
skipLogger_callback = SkipLoggerCallback(n=79)
clr_callback = CyclicLR(mode=CLR_METHOD, base_lr=MIN_LR, max_lr=MAX_LR,step_size= STEP_SIZE * (train_batches.samples/BATCH_SIZE))
callback_list = [skipLogger_callback,clr_callback]
start = time.time()
history = model.fit_generator(generator = train_batches, steps_per_epoch=train_batches.samples/BATCH_SIZE, 
                              validation_data = validation_batches, validation_steps = validation_batches.samples/BATCH_SIZE, 
                              epochs=EPOCHS, verbose=0, workers=4, callbacks = callback_list)

print('time taken : %.4f' % (time.time() - start))
oos_test_loss, oos_test_accuracy = get_oos_test_accuracy(model)
print('OOS Test loss: %.4f OOS std dev: %.4f' % (oos_test_loss, oos_test_accuracy))    
test_loss, test_accuracy = model.evaluate_generator(test_batches, workers=4, verbose=0) 
print('Test loss: %.4f accuracy: %.4f' % (test_loss, test_accuracy))    
history_dict = {}
history_dict['traingular'] = history<jupyter_output><empty_output><jupyter_text>Decaying Triangle schdule<jupyter_code>MIN_LR = 1e-4
MAX_LR = 2e-3
STEP_SIZE = 8
CLR_METHOD = "triangular2"
EPOCHS = 79
sgd_optimizer = optimizers.SGD(lr=MIN_LR, momentum=0.9, nesterov=True)
model.compile(loss="binary_crossentropy", optimizer=sgd_optimizer, metrics=["accuracy"])
skipLogger_callback = SkipLoggerCallback(n=79)
clr_callback = CyclicLR(mode=CLR_METHOD, base_lr=MIN_LR, max_lr=MAX_LR,step_size= STEP_SIZE * (train_batches.samples/BATCH_SIZE))
callback_list = [skipLogger_callback,clr_callback]
start = time.time()
history = model.fit_generator(generator = train_batches, steps_per_epoch=train_batches.samples/BATCH_SIZE, 
                              validation_data = validation_batches, validation_steps = validation_batches.samples/BATCH_SIZE, 
                              epochs=EPOCHS, verbose=0, workers=4, callbacks = callback_list)

print('time taken : %.4f' % (time.time() - start))
oos_test_loss, oos_test_accuracy = get_oos_test_accuracy(model)
print('OOS Test loss: %.4f OOS std dev: %.4f' % (oos_test_loss, oos_test_accuracy))    
test_loss, test_accuracy = model.evaluate_generator(test_batches, workers=4, verbose=0) 
print('Test loss: %.4f accuracy: %.4f' % (test_loss, test_accuracy))
history_dict['triangular2'] = history<jupyter_output><empty_output><jupyter_text>Phase 4 Slow learning with decay<jupyter_code>MAX_EPOCHS = 2000
iteration = int(train_batches.samples/BATCH_SIZE)*MAX_EPOCHS
learning_rate_start = 0.00050
learning_rate_end = 1e-5
decay = ((learning_rate_start/learning_rate_end) - 1)/iteration
print('computed decay : %.2e' % decay)
sgd_optimizer = optimizers.SGD(lr=0.00050, decay=4.30e-04,momentum=0.9, nesterov=True)
model.compile(loss="binary_crossentropy", optimizer=sgd_optimizer, metrics=["accuracy"])
EPOCHS = 200
skipLogger_callback = SkipLoggerCallback(n=25)
start = time.time()
history = model.fit_generator(generator = train_batches, steps_per_epoch=train_batches.samples/BATCH_SIZE, 
                              validation_data = validation_batches, validation_steps = validation_batches.samples/BATCH_SIZE, 
                              epochs=EPOCHS, verbose=0, workers=4, callbacks = callback_list)

print('time taken : %.4f' % (time.time() - start))
oos_test_loss, oos_test_accuracy = get_oos_test_accuracy(model)
print('OOS Test loss: %.4f OOS std dev: %.4f' % (oos_test_loss, oos_test_accuracy))    
test_loss, test_accuracy = model.evaluate_generator(test_batches, workers=4, verbose=0) 
print('Test loss: %.4f accuracy: %.4f' % (test_loss, test_accuracy))
history_dict['simple_decay'] = history
model.save("model/model_mix_lr.h5")
plt.figure(num=None, figsize=(16, 6))

#plt.plot(history_step_decay.history['loss'], label= '.train')
plt.plot(history_step_decay.history['val_loss'], label= 'step_decay', alpha = 0.4)
label_x = [i for i in range(0,len(history_dict['traingular'].history['val_loss']))]
plt.plot(label_x, history_dict['traingular'].history['val_loss'], label= 'triangular-phase1')
label_x2 = [i for i in range(label_x[-1],label_x[-1] + len(history_dict['triangular2'].history['val_loss']))]
plt.plot(label_x2, history_dict['triangular2'].history['val_loss'], label= 'triangular2-phase2')
label_x3 = [i for i in range(label_x2[-1],label_x2[-1] + len(history_dict['simple_decay'].history['val_loss']))]
plt.plot(label_x3, history_dict['simple_decay'].history['val_loss'], label= 'triangular2-phase3')

plt.legend()
plt.title('Training and validation loss vs batch size')
plt.show()
test_batches.reset()
actual_labels = []
X_test = []
i = 0
max_iter = test_batches.samples
for data, label in test_batches:
    actual_labels.append(label)
    X_test.append(data)
    i += 1
    if i == max_iter:
        break
        
actual_labels = np.concatenate(actual_labels)
X_test = np.concatenate(X_test)
test_batches.reset()
predictions = model.predict_generator(test_batches)
test_prediction_data = pd.DataFrame(np.hstack([predictions,actual_labels]).astype("float16"), columns=['predicted_cap','predicted_shoes','actual cap','actual_shoes'])
test_prediction_data[:5]
top_wrong_caps_ix = test_prediction_data[test_prediction_data['actual cap'] == 1]['predicted_cap'].sort_values().index.values[:5]
top_wrong_caps_values = test_prediction_data[test_prediction_data['actual cap'] == 1]['predicted_cap'].sort_values().values[:5]
plot_images(X_test[top_wrong_caps_ix],titles=top_wrong_caps_values)
top_correct_caps_ix = test_prediction_data[test_prediction_data['actual cap'] == 1]['predicted_cap'].sort_values(ascending=False).index.values[:5]
top_correct_caps_values = test_prediction_data[test_prediction_data['actual cap'] == 1]['predicted_cap'].sort_values(ascending=False).values[:5]
plot_images(X_test[top_correct_caps_ix],titles=top_correct_caps_values)
top_wrong_shoes_ix = test_prediction_data[test_prediction_data['actual_shoes'] == 1]['predicted_shoes'].sort_values().index.values[:5]
top_wrong_shoes_values = test_prediction_data[test_prediction_data['actual_shoes'] == 1]['predicted_shoes'].sort_values().values[:5]
plot_images(X_test[top_wrong_shoes_ix],titles=top_wrong_shoes_values)
top_correct_shoes_ix = test_prediction_data[test_prediction_data['actual_shoes'] == 1]['predicted_shoes'].sort_values(ascending=False).index.values[:5]
top_correct_shoes_values = test_prediction_data[test_prediction_data['actual_shoes'] == 1]['predicted_shoes'].sort_values(ascending=False).values[:5]
plot_images(X_test[top_correct_shoes_ix],titles=top_correct_shoes_values)
labels = ['cap','shoes']
cm = confusion_matrix(np.argmax(actual_labels ,axis=1), np.argmax(predictions, axis=1), [0,1])

ax= plt.subplot()
sns.heatmap(cm, annot=True, ax = ax);

ax.set_xlabel('Predicted labels');
ax.set_ylabel('True labels'); 
ax.set_title('Confusion Matrix'); 
ax.xaxis.set_ticklabels(labels); ax.yaxis.set_ticklabels(labels);
fpr, tpr, thresholds = roc_curve(actual_labels[:,1], predictions[:,1])
roc_auc = auc(fpr, tpr)

plt.figure(figsize=(7.5, 7.5))
plt.plot(fpr, tpr, color='darkorange', lw=1, label='ROC curve (area = %0.2f)' % roc_auc)
plt.plot([0, 1], [0, 1], color='navy', lw=2, linestyle='--')
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic')
plt.legend(loc="lower right")
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12561523646479777, 'fraction_numerical': 0.02075754333404665, 'mean_word_length': 5.395985401459854, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '270168', 'license': 'Apache-2.0', 'n_tokens_mistral': 5079, 'n_tokens_neox': 4851, 'n_words': 769, 'path': '4_9_Comparison of step decay and alternative schedule.ipynb', 'repo_name': 'yantraguru/deeplearn', 'size': '14019'}