<jupyter_start><jupyter_text>Evolution Algorithm<NAME> | SPRINGBOARD ObjectiveThe purpose of this notebook is to automate the hyperparameter searching process for my super-resolution model. The goal would be to leave a setup running in a continous loop that randomly generates hyperparameter values (within reason), trains the model, and saves the results. After a few days of iterations, I can compare the results and choose the best model.The setup is designed to run in google colab using their provided GPU. In the case of an exhaustion error, the hyperparameters are scrapped and the iteration is rerun with new hyperparameters.  Helper functionsThe functions below are helper functions used to execute the continous loop at the bottom. The elements involved are:- a set of hyperparamters `h`- a function `randomize_hyperparameters` to choose random hyperparameters- a function `residual_block` which returns the residual block component of the model architecture- a function `conv_block` which returns the convolutional blocks at the end of the model architecture- a function `upsample` which returns the 2x upsample block- a function `build_model` which returns the model using the components above- a function `build_dataset` which loads the dataset into memory- a function `get_filenames` used for fetching image filenames- two functions `display_results` and `display_error_plots` for printing plots onto the screen- two functions `save_results` and `save_error_plots` for saving the plots into local drive<jupyter_code>from google.colab import files

import keras
from keras.layers import Conv2D, UpSampling2D, Input, Add
from keras.models import Model
from keras.regularizers import l1_l2
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
from PIL import Image
from numpy.random import randint
import numpy as np
import os

# hyperparameter
h = {
	"l1_parameter" : 0.01,		# for l1 regularizer. [0, 0.02]
	"l2_parameter" : 0.01,		# for l2 regularizer. [0, 0.02]
	"num_residual_blocks" : 4,	# number of residual blocks to use. [2,10]
	"num_conv_blocks" : 2, 		# number of conv blocks to use inside a residual block. [2,4]
	"num_final_conv_blocks" : 2,
	"num_epochs" : 100,			# train everything at 100 epochs for now
	"batch_size" : 16,			# lower this number if ResourceExhaustion errors occur
	"num_filters" : 64,			# [16,32,64,128]
	"learning_rate" : 0.0001,	# parameter for adam optimizer. [0.0001, 0.001]
	"beta_1" : 0.9,				# parameter for adam optimizer. ignore for now
	"beta_2" : 0.999			# parameter for adam optimizer. ignore for now
}
h.update({
	"optimizer" : keras.optimizers.Adam(lr=h['learning_rate'], beta_1=h['beta_1'], beta_2=h['beta_2'], amsgrad=False),
	'regularizer' : l1_l2(l1=h['l1_parameter'], l2=h['l2_parameter'])
})

# randomizes global variable h
def randomize_hyperparamters():
	global h
	# Note: N evenly spaced random points in interval [a,b) is given by:
	# a + (b - a) * randint(N)/N
	h.update({'l1_parameter' : 0.02 * randint(10) / 10,
			  'l2_parameter' : 0.02 * randint(10) / 10,
			  'num_residual_blocks' : np.random.choice([8,9,10,11,12,13,14,15]),
			  'num_conv_blocks' : np.random.choice([2,3,4]),
			  'num_final_conv_blocks' : np.random.choice([2,3,4]),
			  'num_filters' : np.random.choice([16,32,64,128]),
			  'learning_rate' : 0.00005 + (0.001 - 0.00005) * randint(20) / 20
		})
	h.update({"optimizer" : keras.optimizers.Adam(lr=h['learning_rate'], beta_1=h['beta_1'], beta_2=h['beta_2'], amsgrad=False),
			  'regularizer' : l1_l2(l1=h['l1_parameter'], l2=h['l2_parameter'])})

# a residual block
def residual_block(input_layer, activation='relu', kernel_size=(3,3)):
	global h
	layer = input_layer
	for i in range(h['num_conv_blocks']):
		layer = Conv2D(h['num_filters'], kernel_size, padding='same', activation=activation, activity_regularizer=h['regularizer'])(layer)
	conv_1x1 = Conv2D(3, (1,1), padding='same')(layer)
	return Add()([conv_1x1, input_layer])

# final convolution blocks
def conv_block(input_layer, kernel_size=(3,3)):
	global h
	layer = input_layer
	for i in range(h['num_final_conv_blocks']):
		layer = Conv2D(h['num_filters'], kernel_size, padding='same', activation='relu')(layer)
	return layer

# upsamples 2x
def upsample(layer):
	return UpSampling2D(size=(2,2))(layer)

# builds model based on hyperparameter specs
def build_model():
  global h
  input_layer = Input(shape=(150,150,3))
  layer = input_layer
  # layer = residual_block(input_layer, h['num_filters'])
  for i in range(h['num_residual_blocks']):
    layer = residual_block(layer)
  layer = upsample(layer)
  layer = conv_block(layer)
  output_layer = Conv2D(3, (1,1), padding='same')(layer)
  return Model(inputs=input_layer, outputs=output_layer)

# returns dataset in (x_train, y_train), (x_test, y_test) format
def build_dataset(directory):
	# initialize variables
	filenames = get_filenames(directory)
	X = []
	Y = []

	# collect images from directory
	for filename in filenames:
		print("Processing", filename)
		image = Image.open(directory + filename)
		image_large = np.array(image)
		image_small = np.array(image.resize((150,150)))
		Y.append(image_large)
		X.append(image_small)

	# convert to matrices
	X = np.asarray(X)
	X = X.astype('float32')
	X /= 255
	Y = np.asarray(Y)
	Y = Y.astype('float32')
	Y /= 255

	x_train, x_test, y_train, y_test = train_test_split(X, Y, test_size=0.2, random_state=0)

	# following keras convention for load_data() 
	return ((x_train, y_train), (x_test, y_test))

# helper function for getting image file names
def get_filenames(directory):
	for _,_,filenames in os.walk(directory):
		pass
	return filenames

# show results of a trained model 
def display_results(model, n=10, img_size=10):
	if n > 1:
		fig, axs = plt.subplots(n, 3, figsize=(3 * img_size, n * img_size))
		for index, ax in enumerate(axs):
			ax[0].imshow(x_test[index])
			ax[0].axis('off')
			ax[1].imshow(y_test[index])
			ax[1].axis('off')
			y_pred = model.predict(x_test[index:index+1])[0]
			ax[2].imshow(y_pred)
			ax[2].axis('off')
		plt.show()
	else:
		print("n must be at least 2")

# plot accuracy and error over epochs
def display_error_plots(hist):
	if type(hist) == keras.callbacks.History:
		acc = hist.history['acc']
		val_acc = hist.history['val_acc']
		loss = hist.history['loss']
		val_loss = hist.history['val_loss']

		fig, axs = plt.subplots(1,2, figsize=(15,5))
		axs[0].plot(acc, label='Training accuracy')
		axs[0].plot(val_acc, label='Validation accuracy')
		axs[0].legend(loc='lower right')
		axs[0].set_title("Accuracy")
		axs[1].plot(loss, label='Training error')
		axs[1].plot(val_loss, label='Validation error')
		axs[1].legend()
		axs[1].set_title("Error")
		plt.show()
	else:
		raise TypeError("Expected object of type keras.callbacks.History not " + type(hist).__name__)
  
# save error and accuracy plots
def save_error_plots(hist):
	acc = hist.history['acc']
	val_acc = hist.history['val_acc']
	loss = hist.history['loss']
	val_loss = hist.history['val_loss']

	fig, axs = plt.subplots(1,2, figsize=(15,5))
	axs[0].plot(acc, label='Training accuracy')
	axs[0].plot(val_acc, label='Validation accuracy')
	axs[0].legend(loc='lower right')
	axs[0].set_title("Accuracy")
	axs[1].plot(loss, label='Training error')
	axs[1].plot(val_loss, label='Validation error')
	axs[1].legend()
	axs[1].set_title("Error")

	plt.savefig('plots.png')

# save final images comparing results
def save_results(model):
  fig, axs = plt.subplots(10, 3, figsize=(30,100))
  for index, ax in enumerate(axs):
    ax[0].imshow(x_test[index])
    ax[0].axis('off')
    ax[1].imshow(y_test[index])
    ax[1].axis('off')
    y_pred = model.predict(x_test[index:index+1])[0]
    ax[2].imshow(y_pred)
    ax[2].axis('off')
  plt.savefig('results.png')<jupyter_output>Using TensorFlow backend.<jupyter_text>Continous LoopThe setup below runs indefintely in a loop until manually stopped.In each iteration, a model is trained with randomly chosen hyperparameters. At the end of the iteration, the model architecture, hyperparameters, weights, and plots are saved and stored in a zip file.<jupyter_code># load google drive
from google.colab import drive
drive.mount('/content/gdrive')
# get dataset
directory = 'gdrive/My Drive/Projects/Data Science/Super Resolution/dataset/downscaled/'
(x_train, y_train), (x_test, y_test) = build_dataset(directory)
import time
import contextlib
import json

# ignore ugly tensorflow deprecation warnings
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning) 

# h.update({'num_epochs' : 2}) # for testing only

evolution_iteration = 0
failed_attempts = 0
while(True):
  evolution_iteration += 1
  print("Iteration:", evolution_iteration)
  print("Total failed attempts:", failed_attempts)
  print("---------------------------------")
  print("Randomizing hyperparameters...")

  try:
    # pick random parameters
    randomize_hyperparamters()
    print('residual blocks:', h['num_residual_blocks'])
    print('conv blocks:', h['num_conv_blocks'])
    print('final conv blocks:', h['num_final_conv_blocks'])
    print('num_filters:', h['num_filters'])
    print('learning rate:', h['learning_rate'])
    print('l1:', h['l1_parameter'])
    print('l2:', h['l2_parameter'])
    print("---------------------------------")

    # build, compile and train model
    # NOTE: changed loss function from mse to mae on Dec 7, 2019
    model = build_model()
    model.compile(loss='mae', optimizer=h['optimizer'], metrics=['accuracy'])
    hist = model.fit(x_train, y_train, batch_size=h['batch_size'], epochs=h['num_epochs'], verbose=1, validation_data=(x_test, y_test))

    # record accuracy
    accuracy = model.evaluate(x_test, y_test)[1]

    # save stats by redirecting stdout to a file
    print("\nSaving files...")
    import sys
    orig_stdout = sys.stdout
    f = open('model_summary.txt', 'w')
    sys.stdout = f
    model.summary()   # save model 
    print("\nhyperparameters:", h, "\n") # save hyperparameters
    print("final validation accuracy:", accuracy) # save accuracy
    sys.stdout = orig_stdout
    f.close()

    print("Saving plots...")
    with contextlib.redirect_stdout(None): # bypass ugly matplotlib clipping errors
      save_error_plots(hist) # save accuracy and error plots
      save_results(model) # save image results

    # save model architecture
    print("Saving model architecture...")
    model_json = model.to_json()
    with open('model.json', 'w') as f:
      json.dump(model_json, f)

    # zip up files
    print("Zipping files...")
    !zip {str(evolution_iteration)}.zip model_summary.txt results.png plots.png results.png model.json

    # store the zipped file
    print("Storing files in designated folder...")
    time.sleep(5)   # wait 5 seconds to ensure zipping is complete
    !mv {str(evolution_iteration)}.zip gdrive/'My Drive'/Projects/'Data Science'/'Super Resolution'/final_models/

    print("Iteration", evolution_iteration, "successful.\n")
  except:
    # try again with new parameters if running into resource issuses
    print("Attempt Failed\n")
    failed_attempts += 1<jupyter_output>Iteration: 1
Total failed attempts: 0
---------------------------------
Randomizing hyperparameters...
residual blocks: 14
conv blocks: 4
final conv blocks: 4
num_filters: 32
learning rate: 0.0008100000000000001
l1: 0.018
l2: 0.018
---------------------------------
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:66: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.

WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:541: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.

WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4432: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.

WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:2239: The name tf.image.resize_nearest_neighbor is deprecate[...]
{'fraction_non_alphanumeric': 0.13691164327002478, 'fraction_numerical': 0.0282411230388109, 'mean_word_length': 4.638268156424581, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '611818', 'license': 'MIT', 'n_tokens_mistral': 4324, 'n_tokens_neox': 3997, 'n_words': 1083, 'path': '5_evolution_algorithm.ipynb', 'repo_name': 'salman-a-shah/Springboard', 'size': '12115'}