<jupyter_start><jupyter_text>Import Dataset<jupyter_code>def load_cup():
    ml_cup = np.delete(np.genfromtxt('ML-CUP20-TR.csv', 
                                     delimiter=','), obj=0, axis=1)
    return ml_cup[:, :-2], ml_cup[:, -2:]

def load_cup_blind():
    return np.delete(np.genfromtxt('ML-CUP20-TS.csv',
                                   delimiter=','), obj=0, axis=1)
    
def mean_euclidean_error(y_true, y_pred):
    assert y_true.shape == y_pred.shape
    return np.mean(np.linalg.norm(y_pred - y_true, axis=1))
X, y = load_cup()

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
print(X.shape)
print(y.shape)
# data = np.genfromtxt('ML-CUP20-TR.csv', delimiter=',', dtype=np.float64)
# X = data[:, 1:-2]
# y = data[:, -2:]

# print(X.shape)
# print(y.shape)
# Xtrain, Xval, ytrain, yval = train_test_split(X, y, test_size=0.20, random_state=42)
# print(Xtrain.shape)
# print(ytrain.shape)
# print(Xval.shape)
# print(yval.shape)<jupyter_output>time: 930 µs (started: 2021-01-21 08:11:24 +00:00)<jupyter_text>Grid Search - Gradient Descent<jupyter_code>txt_best_grids = "best_results_cup_nn.txt"
grid_results_name = 'grid_results_cup_nn_v1.csv'
from sklearn.model_selection import GridSearchCV
from tensorflow.keras.models import Sequential
from tensorflow.keras import layers
from tensorflow.keras.layers import Dense
from tensorflow.keras.wrappers.scikit_learn import KerasRegressor
from tensorflow.keras.optimizers import SGD
from tensorflow.keras.regularizers import L2

# Function to create model, required for KerasRegressor
def create_model(learn_rate=0.01, momentum=0, neurons_layer_1=10,
                 init_mode='uniform', activation='relu',  weight_decay = 0.0,
                 nesterov = False, neurons_layer_2 = 5):
  # create model
  model = Sequential()
  model.add(layers.InputLayer(input_shape=(10)))
  model.add(Dense(neurons_layer_1, activation=activation, kernel_initializer=init_mode, kernel_regularizer= L2(weight_decay)))
  model.add(Dense(neurons_layer_2, activation=activation, kernel_initializer=init_mode, kernel_regularizer= L2(weight_decay)))
  model.add(Dense(2))
  optimizer = SGD(
      learning_rate=learn_rate, momentum=momentum, nesterov=nesterov)
  # Compile model
  model.compile(optimizer=optimizer,
              loss=euclidean_distance_loss)
  return model

BATCH_SIZE = len(X_train)

# create model
model = KerasRegressor(build_fn=create_model, verbose=0) #verbose = 0
# define the grid search parameters
batch_size = [64]#, 128, 256, BATCH_SIZE]
epochs = [300]
learn_rate = [0.001]#, 0.01, 0.1, 0.3]
momentum = [0.0, 0.2]#, 0.4, 0.9]
nesterov = [False]
neurons_layer_1 = [40]#, 80, 120]
neurons_layer_2 = [40]#, 80, 120]
init_mode = ['glorot_uniform']
activation = ['relu'] #, 'tanh', 'sigmoid']
weight_decay = [0.1]#, 0.01, 0.001, 0.0001]

param_grid = dict(batch_size=batch_size, epochs=epochs, learn_rate=learn_rate,
                  momentum=momentum, neurons_layer_1=neurons_layer_1, 
                  init_mode=init_mode, activation=activation, weight_decay= weight_decay,
                  nesterov= nesterov, neurons_layer_2=neurons_layer_2)
grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3, return_train_score=True, verbose=True)
grid_result = grid.fit(X_train, y_train)

import json

grid_results_name_str = f'Results wrt: {grid_results_name} \n'
grid_params_str = f'Grid on: {json.dumps(grid_result.param_grid)} \n'
grid_results_str = f'Best: {grid_result.best_score_} using {grid_result.best_params_} \n'

with open(txt_best_grids, "a") as file_object:
    file_object.write(grid_results_name_str)
    file_object.write(grid_params_str)
    file_object.write(grid_results_str)
    file_object.write('\n')
print(grid_results_str)
import pandas as pd


df = pd.DataFrame(grid.cv_results_)[['rank_test_score','mean_test_score',
                                     'std_test_score','mean_train_score',
                                     'std_train_score','param_activation',
                                     'param_batch_size','param_epochs',
                                     'param_init_mode', 'param_learn_rate',
                                     'param_momentum','param_nesterov',
                                     'param_neurons_layer_1', 'param_neurons_layer_2',
                                     'param_weight_decay','mean_fit_time']].sort_values(by='rank_test_score')

df.rename(columns={'param_activation': 'activation',
                   'param_batch_size': 'batch_size',
                   'param_epochs': 'epochs',
                   'param_init_mode': 'init_mode',
                   'param_learn_rate': 'learn_rate',
                   'param_momentum': 'momentum',
                   'param_init_mode': 'init_mode',
                   'param_weight_decay': 'weight_decay',
                   'param_nesterov': 'nesterov',
                   'param_neurons_layer_1': 'neurons_layer_1',
                   'param_neurons_layer_2': 'neurons_layer_2',
                   'mean_test_score': 'mean_val_score',
                   'std_test_score': 'std_val_score',                
                   'rank_test_score': 'rank_val_score'}, inplace=True)

df.mean_train_score *= -1
df.mean_val_score *= -1
df
df.to_csv(grid_results_name)
# means = grid_result.cv_results_['mean_test_score']
# stds = grid_result.cv_results_['std_test_score']
# params = grid_result.cv_results_['params']
# for mean, stdev, param in zip(means, stds, params):
#     print("%f (%f) with: %r" % (mean, stdev, param))

# import csv

# with open(single_grid_results_filename, mode='w') as grid:
#     grid = csv.writer(grid, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
#     for mean, stdev, param in zip(means, stds, params):
#       grid.writerow([mean, stdev, param])
# from sklearn.externals import joblib

# # #save your model or results
# # joblib.dump(grid_result, 'gs_object_monk3.pkl')
# # #load your model for further usage
# # boh = joblib.load("gs_object_monk3.pkl")

# # joblib.dump(grid.best_estimator_, 'gs_best_estimator_monk3.pkl', compress = 1)
# # filename = 'finalized_model.sav'
# # pickle.dump(model, open(filename, 'wb'))<jupyter_output>time: 5.94 ms (started: 2021-01-20 17:45:32 +00:00)
{'fraction_non_alphanumeric': 0.12648902821316615, 'fraction_numerical': 0.026175548589341693, 'mean_word_length': 3.2035573122529644, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '604154', 'license': 'MIT', 'n_tokens_mistral': 2391, 'n_tokens_neox': 2237, 'n_words': 383, 'path': 'src/keras/CUP_Keras_NN_GS.ipynb', 'repo_name': 'luca-demartino/machine-learning-project', 'size': '6380'}