<jupyter_start><jupyter_text>Fine-Tuning Neural Network Model This notebook assumes the data has been split into a training and a test set. If not, run get_data.ipynb first.<jupyter_code>import pandas as pd

TRAINING_FILEPATH = 'data/training_set.csv'
TEST_FILEPATH = 'data/test_set.csv'

training_set = pd.read_csv(TRAINING_FILEPATH, index_col='index')
test_set = pd.read_csv(TEST_FILEPATH, index_col='index')
from preprocessing_utils import stratify_split_imdb

# create a validation set to evaluate models instead of using 5-fold cross validation which is time-consuming for neural networks
small_training_set, validation_set = stratify_split_imdb(training_set, test_size=0.2)
print(small_training_set.shape, validation_set.shape)
from preprocessing_utils import FeaturePreprocessor, separate_features_targets

train_X, train_y = separate_features_targets(small_training_set)
val_X, val_y = separate_features_targets(validation_set)

# preprocess training features (add combinations, power transform)
preprocessor = FeaturePreprocessor(add_combinations=True, powertransform_num=True, onehot_type=True)
train_X = preprocessor.fit_transform(train_X)
val_X = preprocessor.transform(val_X)
# TODO: create a base neural network model
# TODO: train the base model
#   TODO: use tensorboard to visualize training and validation error
# TODO: fine-tune the model using random search and grid search
# TODO: use early stopping to train the best model
import tensorflow as tf
import tensorflow.keras as keras

def create_neuralnet(input_shape, n_hidden=1, n_neurons=32, hidden_activation='relu'):
    model = keras.models.Sequential()
    
    # input layer
    model.add(keras.layers.Dense(n_neurons, input_shape=input_shape, activation=hidden_activation))
    
    # hidden layers
    for i in range(n_hidden-1):
        model.add(keras.layers.Dense(n_neurons, activation=hidden_activation))
    
    # output layer
    model.add(keras.layers.Dense(1))

    # compile the model
    # TODO: make the optimizer or SGD learning rate a hyperparameter
    model.compile(loss='mean_squared_error', optimizer='sgd', metrics=[tf.keras.metrics.RootMeanSquaredError(name='rmse')])
    
    return model
from train_utils import get_fit_logdir

# generate tensorboard logs
logdir = get_fit_logdir()
tensorboard_cb = keras.callbacks.TensorBoard(logdir)

# create the base model
base_neuralnet = create_neuralnet(input_shape=train_X.shape[1:], n_hidden=1, n_neurons=32, hidden_activation='relu')

# train the model
base_neuralnet.fit(train_X, train_y, batch_size=32, epochs=150, validation_data=(val_X, val_y), callbacks=[tensorboard_cb])
from train_utils import evaluate_keras_model

base_neuralnet_errors = evaluate_keras_model(base_neuralnet, train_X, train_y, val_X, val_y)
base_neuralnet_errors
import os
import time
import numpy as np
from train_utils import TENSORBOARD_LOGS_DIR
from train_utils import evaluate_keras_model
from sklearn.model_selection import ParameterGrid


def neuralnet_random_search(param_grid, n_iter, train_X, train_y, val_X, val_y, verbose=False):
    """
    Randomized hyperparameter search for neural network models
    """

    param_grid = ParameterGrid(param_grid)
    grid_perms = np.random.permutation(param_grid)[:n_iter]

    base_log_dir = os.path.join(TENSORBOARD_LOGS_DIR, "rndsearch_" + time.strftime("%Y_%m_%d-%H_%M_%S"))
    search_results = {
        'params': [],
        'best_val_rmse': float('inf'),
        'best_index': None,
        'best_model': None,
        'best_params': None
    }

    if verbose:
        print(f"Fitting {n_iter} models\n")

    for i, params in enumerate(grid_perms):
        # generate tensorboard logs
        logdir = os.path.join(base_log_dir, f"fit_{i}")
        tensorboard_cb = keras.callbacks.TensorBoard(logdir)
        cb = [tensorboard_cb]

        # create the model
        model = create_neuralnet(input_shape=train_X.shape[1:],
                                 n_hidden=params['n_hidden'],
                                 n_neurons=params['n_neurons'],
                                 hidden_activation=params['hidden_activation'])

        if verbose:
            print(f"Fit {i}\t{params}")
            print(f"Logging to {logdir}")

        # fit the model to the data
        model.fit(train_X, train_y,
                  batch_size=params['batch_size'],
                  epochs=params['epochs'],
                  validation_data=(val_X, val_y),
                  callbacks=cb,
                  verbose=(0 if not verbose else 1))
        
        # evaluate the model
        errors = evaluate_keras_model(model, train_X, train_y, val_X, val_y)
        train_rmse = errors['train_rmse'][0]
        val_rmse = errors['val_rmse'][0]

        # update search results
        param_entry = params.copy()
        param_entry['train_rmse'] = train_rmse
        param_entry['val_rmse'] = val_rmse
        param_entry['logdir'] = logdir
        search_results['params'].append(param_entry)

        # check if this is the best model so far
        if val_rmse < search_results['best_val_rmse']:
            search_results['best_val_rmse'] = val_rmse
            search_results['best_index'] = i
            search_results['best_model'] = model
            search_results['best_params'] = params
        
    return search_results
neuralnet_rndsearch_grid = {
    'n_hidden': [1, 2, 4],
    'n_neurons':[16, 32],
    'hidden_activation': ['relu', 'sigmoid'],
    'epochs': [150],
    'batch_size': [4, 8, 16, 32, 64, 128],
}

rndsearch_results = neuralnet_random_search(neuralnet_rndsearch_grid, 15, train_X, train_y, val_X, val_y, verbose=True)
import os
from train_utils import MODELS_DIR

# save the best model
filepath = os.path.join(MODELS_DIR, 'rndsearch_neuralnet.h5')
rndsearch_results['best_model'].save(filepath)
pd.DataFrame(rndsearch_results['params']).sort_values(by=['batch_size', 'n_hidden', 'n_neurons'])
# Good hyperparameters (decided by examining error plots for different models)
# - batch_size=64
# - n_hidden=2
# - n_neurons=16
# - hidden_activation='relu'
from keras.callbacks import EarlyStopping

# generate tensorboard logs
logdir = get_fit_logdir()
tensorboard_cb = keras.callbacks.TensorBoard(logdir)

# early stopping callback
earlystopping_cb = EarlyStopping(monitor='val_rmse', mode='min', patience=20, min_delta=0.00005, restore_best_weights=True)

# train a model with good hyperparameters
good_neuralnet = create_neuralnet(input_shape=train_X.shape[1:], n_hidden=2, n_neurons=16, hidden_activation='relu')

# train the model
good_neuralnet.fit(train_X, train_y, batch_size=64, epochs=1000,
                   validation_data=(val_X, val_y), callbacks=[tensorboard_cb, earlystopping_cb])
evaluate_keras_model(good_neuralnet, train_X, train_y, val_X, val_y)<jupyter_output><empty_output><jupyter_text>This seems like the best model so far<jupyter_code>from sklearn.metrics import mean_squared_error

# preprocess the test set
test_X, test_y = separate_features_targets(test_set)
test_X = preprocessor.transform(test_X)

# evaluate this model on the test set
test_pred_y = good_neuralnet.predict(test_X)
test_rmse = np.sqrt(mean_squared_error(test_y, test_pred_y))
print(test_rmse)
# save the best model
best_neuralnet = good_neuralnet
filepath = os.path.join(MODELS_DIR, 'best_neuralnet.h5')
best_neuralnet.save(filepath)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09580105992662047, 'fraction_numerical': 0.009512161978529691, 'mean_word_length': 4.337200870195794, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '415467', 'license': 'MIT', 'n_tokens_mistral': 2476, 'n_tokens_neox': 2386, 'n_words': 523, 'path': 'finetuning_neuralnet.ipynb', 'repo_name': 'MoazAshraf/predicting_imdb_ratings', 'size': '7359'}