<jupyter_start><jupyter_text>Import Dataset<jupyter_code>Xtrain, Xtest, ytrain, ytest = load_monk("2")
print(Xtrain.shape)
print(Xtest.shape)
print(ytrain.shape)
print(ytest.shape)<jupyter_output>(124, 17)
(477, 17)
(124, 1)
(477, 1)
time: 20.5 ms (started: 2021-01-17 09:47:25 +00:00)<jupyter_text>Grid Search - Gradient Descent<jupyter_code>best_results_filename = "best_results_monk2.txt"
single_grid_results_filename = 'grid_results_monk2.1.csv'
from sklearn.model_selection import GridSearchCV
from tensorflow.keras.models import Sequential
from tensorflow.keras import layers
from tensorflow.keras.layers import Dense
from tensorflow.keras.wrappers.scikit_learn import KerasClassifier
from tensorflow.keras.optimizers import SGD

# L2 = tf.keras.regularizers.L2(l2=0.02)

# Function to create model, required for KerasClassifier
def create_model(learn_rate=0.01, momentum=0, neurons=1, init_mode='uniform', activation='relu'):
  # create model
  model = Sequential()
  model.add(layers.InputLayer(input_shape=(17)))
  model.add(Dense(neurons, activation=activation, kernel_initializer=init_mode))
  model.add(Dense(1, activation='sigmoid', kernel_initializer=init_mode))
  optimizer = SGD(
      learning_rate=learn_rate, momentum=momentum, nesterov=False)
  # Compile model
  model.compile(optimizer=optimizer,
              loss=tf.keras.losses.MSE,
              metrics='accuracy')
  return model

BATCH_SIZE = len(Xtrain)

# create model
model = KerasClassifier(build_fn=create_model, verbose=0)
# define the grid search parameters
batch_size = [32, 64, BATCH_SIZE]
epochs = [1000]
learn_rate = [0.001, 0.01, 0.1]
momentum = [0.0, 0.2, 0.6, 0.9]
neurons = [3, 4, 5]
init_mode = ['glorot_uniform']
activation = ['relu', 'tanh', 'sigmoid']

# momentum = [0.0, 0.2, 0.4, 0.6, 0.8, 0.9]
# neurons = [2, 3, 4, 5]
# init_mode = ['uniform', 'normal', 'zero', 'glorot_uniform']
# optimizer = ['SGD', 'RMSprop', 'Adagrad', 'Adadelta', 'Adam', 'Adamax', 'Nadam']

param_grid = dict(batch_size=batch_size, epochs=epochs, learn_rate=learn_rate, momentum=momentum, neurons=neurons, init_mode=init_mode, activation=activation)#, optimizer=optimizer)
grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)
grid_result = grid.fit(Xtrain, ytrain)

# summarize results
print("Best: %f using %s" % (grid_result.best_score_, grid_result.best_params_))


grid_values_string = f'Grid on: batch_size: {batch_size}, epochs: {epochs}, learn_rate: {learn_rate}, momentum: {momentum}, init_mode = {init_mode} \n'
grid_results_string = f'Best: {grid_result.best_score_} using {grid_result.best_params_} \n'
grid_correspondant_parameter_name = f'Results in: {single_grid_results_filename} \n'
# Open a file with access mode 'a'
with open(best_results_filename, "a") as file_object:
    # Append 'hello' at the end of file
    file_object.write(grid_correspondant_parameter_name)
    file_object.write(grid_values_string)
    file_object.write(grid_results_string)
means = grid_result.cv_results_['mean_test_score']
stds = grid_result.cv_results_['std_test_score']
params = grid_result.cv_results_['params']
for mean, stdev, param in zip(means, stds, params):
    print("%f (%f) with: %r" % (mean, stdev, param))
import csv

with open(single_grid_results_filename, mode='w') as grid:
    grid = csv.writer(grid, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
    for mean, stdev, param in zip(means, stds, params):
      grid.writerow([mean, stdev, param])
# from sklearn.externals import joblib

# # #save your model or results
# # joblib.dump(grid_result, 'gs_object_monk3.pkl')
# # #load your model for further usage
# # boh = joblib.load("gs_object_monk3.pkl")

# # joblib.dump(grid.best_estimator_, 'gs_best_estimator_monk3.pkl', compress = 1)
# # filename = 'finalized_model.sav'
# # pickle.dump(model, open(filename, 'wb'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13356944093093853, 'fraction_numerical': 0.02732102200860106, 'mean_word_length': 5.4188311688311686, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '365816', 'license': 'MIT', 'n_tokens_mistral': 1540, 'n_tokens_neox': 1456, 'n_words': 298, 'path': 'src/keras/Monk2_GS.ipynb', 'repo_name': 'luca-demartino/machine-learning-project', 'size': '3953'}