<jupyter_start><jupyter_text>Part 1 - a simple keras model for the IMDB datasetWe'll use the IMDB movie review classification dataset built into Keras as the problem. First, let's run that dataset through a simple model. (Code based on the Chollet example)<jupyter_code>import numpy as np
from keras.datasets import imdb
from keras import models
from keras import layers
from keras import optimizers
from keras import regularizers

% matplotlib inline
(train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10000)<jupyter_output><empty_output><jupyter_text>Let's decode one of these reviews back into English<jupyter_code>word_index = imdb.get_word_index()
reverse_word_index = dict(
    [(value, key) for (key, value) in word_index.items()])
decoded_review = ' '.join(
    [reverse_word_index.get(i - 3, '?') for i in train_data[0]])
decoded_review
train_data.shape<jupyter_output><empty_output><jupyter_text>Let's prepare the data by padding the lists and turning them into a tensor.<jupyter_code>import numpy as np
def vectorize_sequences (sequences, dimension = 10000):
    results = np.zeros((len(sequences), dimension))
    for i, sequence in enumerate(sequences):
        results[i, sequence] = 1.
    return results
x_train = vectorize_sequences (train_data)
x_test = vectorize_sequences (test_data)
y_train = np.asarray(train_labels).astype('float32')#.reshape(y_train.shape[0], 1)
y_test = np.asarray(test_labels).astype('float32')#.reshape(y_train.shape[0], 1)<jupyter_output><empty_output><jupyter_text>x_train now has dimensions (m, n) where m is number of training examples and n is number of features (10,000 possible word features, in this case)<jupyter_code>print ("x_train.shape = ", x_train.shape)
print ("x_test.shape = ", x_test.shape)
print ("y_train.shape = ", y_train.shape)
print ("y_test.shape = ", y_test.shape)<jupyter_output>x_train.shape =  (25000, 10000)
x_test.shape =  (25000, 10000)
y_train.shape =  (25000,)
y_test.shape =  (25000,)<jupyter_text>As we're going to be doing some quite intensive fitting of hyperparameters based on accuracy of the test set, we need a third dev set. Since the test set is generously sized, let's just split that in two.<jupyter_code>x_dev = x_test[0: 10000,:]
y_dev = y_test[0: 10000]

x_test = x_test[10000:x_test.shape[0], :]
y_test = y_test[10000:y_test.shape[0]]
print ("x_train.shape = ", x_train.shape)
print ("x_dev.shape = ", x_dev.shape)
print ("x_test.shape = ", x_test.shape)

print ("y_train.shape = ", y_train.shape)
print ("y_dev.shape = ", y_dev.shape)
print ("y_test.shape = ", y_test.shape)<jupyter_output>x_train.shape =  (25000, 10000)
x_dev.shape =  (10000, 10000)
x_test.shape =  (15000, 10000)
y_train.shape =  (25000,)
y_dev.shape =  (10000,)
y_test.shape =  (15000,)<jupyter_text>Initialize the network<jupyter_code>def build_model(lambd = 0.001, learning_rate = 0.001, dropout_rate = 0.20):
    
    model = models.Sequential()
    model.add (layers.Dense(16, 
                            kernel_regularizer = regularizers.l2(lambd),
                            activation='relu', input_shape = (10000,)))
    model.add (layers.Dropout(float(dropout_rate))) # dropout_rate gets passed in as (1,) array
                                                    # 
    model.add (layers.Dense(16, 
                            kernel_regularizer = regularizers.l2(lambd),
                            activation='relu'))
    model.add (layers.Dropout(float(dropout_rate)))
    model.add (layers.Dense(1, activation='sigmoid'))
    model.compile (optimizer=optimizers.RMSprop(lr = learning_rate), 
               loss = 'binary_crossentropy',
               metrics = ['acc'])
    
    return (model)
import matplotlib.pyplot as plt
acc = history_dict['acc']
loss_values = history_dict['loss']
val_loss_values = history_dict['val_loss']

epochs = range(1, len(acc) + 1)

plt.plot (epochs, loss_values, 'bo', label = 'Training loss')
plt.plot (epochs, val_loss_values, 'b', label = 'Validation loss')
plt.title('Training and validation loss')
plt.xlabel('Epochs')
plt.ylabel('Loss')
plt.legend()

plt.show
plt.clf()

acc_values = history_dict['acc']
val_acc_values = history_dict['val_acc']

plt.plot (epochs, acc_values, 'bo', label = 'Training acc')
plt.plot (epochs, val_acc_values, 'b', label = 'Validation acc')
plt.title('Training and validation accuracy')
plt.xlabel('Epochs')
plt.ylabel('Accuracy')
plt.legend()

plt.show<jupyter_output><empty_output><jupyter_text>Hyperparameter Search Algorithm<jupyter_code>print ("training accuracy", history.history['acc'][-1])
print ("test accuracy", history.history['val_acc'][-1])
print ("training loss", history.history['loss'][-1])
print ("test loss", history.history['val_loss'][-1])<jupyter_output><empty_output><jupyter_text>To start with, let's build an algorithm that trains a model for 10 epochs and varies a hyperparameter randomly between two start/end points. Let's just use the learning rate for now. We'll use a log uniform random distribution. We can do this quite easily just by rerunning the `model.compile` step.<jupyter_code>from numpy import random

def search_learning_rate(model, 
                         num_models = 10): # how many different values of the lr to try
                                
        
    
# search random values on a log scale to get learning rates
    lr_exponent = (-3 * random.rand(num_models,1)) - 1 # exponent - search between 10^-4 and 10^-1
    learning_rates = 10 ** lr_exponent # lr is a num_iterations x 1 vector
        
    lambd_exponent = (-4 * random.rand(num_models,1)) - 1
    lambdas = 10 ** lambd_exponent
    
    dropout_rand = random.rand(num_models,1)
    dropout_rates = np.round (0.5 * dropout_rand, 2) 
    print (dropout_rates[:10])
    
    # initialize vectors for validation loss
    val_loss = np.zeros ((num_models, 1))
    val_acc = np.zeros ((num_models, 1))
    
    for i in range (num_models):
        
        model = build_model (lambd = lambdas[i], 
                             learning_rate = learning_rates[i], 
                             dropout_rate = dropout_rates[i])
        
        history = model.fit(x_train, y_train, verbose = 0, epochs = 10, batch_size=512, validation_data=(x_dev, y_dev))
        val_loss[i] = history.history['val_loss'][-1]
        val_acc[i] = history.history['val_acc'][-1]
        print (i, "of", num_models, "lr:", learning_rates[i], "lambda:", lambdas[i], "drop %:", dropout_rates[i], "dev loss:", val_loss[i], "dev acc:", val_acc[i])
    return (learning_rates, lambdas, dropout_rates, val_loss, val_acc)
num_models = 100

results_lr, results_lambd, results_val_loss, results_val_acc = search_learning_rate(model, num_models)

print ("learning rates : validation loss : validation accuracy")
for i in range (num_models):
    print (results_lr[i], ":", results_val_loss[i], ":", results_val_acc[i])
iterations = 10                                 # how many different values of the lr to try

# search random values on a log scale to get learning rates
exponent = (-3 * random.rand(iterations,1)) - 1 # exponent - search between 10^-4 and 10^-1
lr = 10 ** exponent 
print (val_loss)<jupyter_output>[[0.]
 [0.]
 [0.]
 [0.]
 [0.]
 [0.]
 [0.]
 [0.]
 [0.]
 [0.]]
{'fraction_non_alphanumeric': 0.12472375690607734, 'fraction_numerical': 0.030248618784530387, 'mean_word_length': 3.84024064171123, 'pattern_counts': {'":': 2, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '394193', 'license': 'MIT', 'n_tokens_mistral': 2606, 'n_tokens_neox': 2383, 'n_words': 634, 'path': 'imdb dataset.ipynb', 'repo_name': 'nicksexton/hyperparameter_search', 'size': '7240'}