<jupyter_start><jupyter_text>Hyperparameter Optimization (and Overfitting)"The Theory Behind Overfitting, Cross Validation, Regularization, Bagging, and Boosting: Tutorial" (<NAME>, <NAME>)https://arxiv.org/abs/1905.12787We already saw `GridSearchCV`. This is an exhaustive approach to optimization and is prone to overfitting.  `RandomSearch` often gives comparable answers but protects, to some degree, overfitting. See "Random Search for Hyper-Parameter Optimization"<NAME>, <NAME>; 13(Feb):281−305, 2012 for a theoretical discussion.There are Bayesian-like approaches as well, that seek to minimize an objective function over a large range of parameters.[`hyperopt`](http://hyperopt.github.io/hyperopt/) is one such popular approach, which is used as `hyperas` in `keras` (see e.g., https://arxiv.org/abs/1801.01596)<jupyter_code>#!pip install hyperopt hyperas
# see http://maxpumperla.com/hyperas/
from hyperopt import Trials, STATUS_OK, tpe
from hyperas import optim
from hyperas.distributions import choice, uniform


def data():
    '''
    Data providing function:

    Make sure to have every relevant import statement included here and return data as
    used in model function below. This function is separated from model() so that hyperopt
    won't reload data for each evaluation run.
    '''
    from tensorflow.keras.datasets import mnist
    from tensorflow.keras.utils import to_categorical

    (X_train, y_train), (X_test, y_test) = mnist.load_data()
    X_train = X_train.reshape(60000, 784)
    X_test = X_test.reshape(10000, 784)
    X_train = X_train.astype('float32')
    X_test = X_test.astype('float32')
    X_train /= 255
    X_test /= 255
    nb_classes = 10
    Y_train = to_categorical(y_train, nb_classes)
    Y_test = to_categorical(y_test, nb_classes)
    return X_train, Y_train, X_test, Y_test


def model(X_train, Y_train, X_test, Y_test):
    '''
    Model providing function:

    Create Keras model with double curly brackets dropped-in as needed.
    Return value has to be a valid python dictionary with two customary keys:
        - loss: Specify a numeric evaluation metric to be minimized
        - status: Just use STATUS_OK and see hyperopt documentation if not feasible
    The last one is optional, though recommended, namely:
        - model: specify the model just created so that we can later use it again.
    '''
    from tensorflow.keras.models import Sequential
    from tensorflow.keras.layers import Dense, Dropout, Activation

    model = Sequential()
    model.add(Dense(512, input_shape=(784,)))
    model.add(Activation('relu'))
    model.add(Dropout({{uniform(0, 1)}}))
    model.add(Dense({{choice([256, 512, 1024])}}))
    model.add(Activation({{choice(['relu', 'sigmoid'])}}))
    model.add(Dropout({{uniform(0, 1)}}))

    model.add(Dense(10))
    model.add(Activation('softmax'))

    model.compile(loss='categorical_crossentropy', optimizer={{choice(['rmsprop', 'adam', 'sgd'])}})

    model.fit(X_train, Y_train,
              batch_size={{choice([64, 128])}},
              nb_epoch=1,
              verbose=1,
              validation_data=(X_test, Y_test))
    acc = model.evaluate(X_test, Y_test, verbose=0)
    print('Test accuracy:', acc)
    return {'loss': -acc, 'status': STATUS_OK, 'model': model}
X_train, Y_train, X_test, Y_test = data()

best_run, best_model = optim.minimize(model=model,
                                      data=data,
                                      algo=tpe.suggest,
                                      max_evals=5, verbose=False,
                                      trials=Trials(),
                                      notebook_name='03_hyperparameter_optimization')
print("Evalutation of best performing model:")
print(best_model.evaluate(X_test, Y_test))
best_run<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1067400996590611, 'fraction_numerical': 0.023865722528193023, 'mean_word_length': 3.358857142857143, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '467518', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1244, 'n_tokens_neox': 1131, 'n_words': 335, 'path': 'Lectures/7_SelfSupervisedAndAnomalies/03_hyperparameter_optimization.ipynb', 'repo_name': 'nmejia10/ml_course', 'size': '3844'}