<jupyter_start><jupyter_text>test high level API<jupyter_code>%load_ext autoreload
%autoreload 2
dataset = np.genfromtxt('../CUP/ML-CUP19-TR_tr_vl.csv',delimiter=',')
X_train = dataset[:1000,:-2]
Y_train = dataset[:1000,-2:]
    
X_val = dataset[1000:,:-2]
Y_val = dataset[1000:,-2:]

param = {
            "n_layer_units": [38], # con questo si specifica la topologia da provare
            "learning_rate": 0.014,
            "momentum": 0.8,
            "nesterov": True,
            "kernel_regularizer": 0.0001,
            "activation": "sigmoid",
            "early_stop": EarlyStopping(0.00009, 20),
            "max_epoch": 100
}
model = MLPRegressor(X_train.shape[1], Y_train.shape[1], random_state = 42, verbose = 0, **param)
print(model.get_params())
model.fit(X_train, Y_train, X_val, Y_val)
model.get_history()["val_loss_mse"][-1]<jupyter_output>The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload
{'n_layer_units': [38], 'activation': 'sigmoid', 'kernel_regularizer': 0.0001, 'batch_size': None, 'max_epoch': 100, 'learning_rate': 0.014, 'momentum': 0.8, 'nesterov': True, 'early_stop': eps: 9e-05, patience: 20}<jupyter_text>Test Kfold class  - generate k-fold<jupyter_code>dataset = np.genfromtxt('../CUP/ML-CUP19-TR_tr_vl.csv',delimiter=',')
X = dataset[:,:-2]
Y = dataset[:,-2:]
print(X.shape)
kf = Kfold(n_splits=5, shuffle=True, random_state=1)
print(kf.get_n_splits())
split = kf.split(X)
print(split)
print(len(split["train"][0]))
print(len(split["val"][0]))<jupyter_output>1082
270<jupyter_text>save and load k-folds<jupyter_code>split = load_data("../CUP/4folds.index")<jupyter_output><empty_output><jupyter_text>test grid search<jupyter_code>def test(grid):
    for p in grid:
        # Always sort the keys of a dictionary, for reproducibility
        items = sorted(p.items())
        if not items:
           yield {}
        else:
            keys, values = zip(*items)
            for v in product(*values):
                params = dict(zip(keys, v))
                yield params

def product_dict(**kwargs):
    keys = kwargs.keys()
    vals = kwargs.values()
    for instance in itertools.product(*vals):
        yield dict(zip(keys, instance))

grid = {
            "n_units": [(10, 20), (101, 201), (1,)], # con questo si specifica la topologia da provare
            "learning_rate": [0.014, 0.017, 0.020],
            "momentum": [0.8],
            "nesterov": [False, True],
            "kernel_regularizer": [0.0001, 0.0002, 0.0003],
            "activation": ["sigmoid", "relu"],
            "early_stop": [es]
}

default_grid = {
            "n_units": [100], # con questo si specifica la topologia da provare
            "activation": ["relu"],
            "kernel_regularizer": [0.0001],

            "batch_size": [None],
            "max_epoch": [1000],
            "learning_rate": [0.014, 0.017, 0.020],
            "momentum": [0.8],
            "nesterov": [False, True],
            "early_stop": [es]
}

print(len(list(product_dict(**grid))))
default_grid.update(grid)
print(default_grid[0])
%load_ext autoreload
%autoreload 2

dataset = np.genfromtxt('../CUP/ML-CUP19-TR_tr_vl.csv',delimiter=',')
split = load_data("../CUP/4folds.index")
X_train = dataset[:,:-2]
Y_train = dataset[:,-2:]

es = EarlyStopping(0.00009, 20)


mlp_r = MLPRegressor(X_train.shape[1], Y_train.shape[1])

grid0 = {
            "n_layer_units": [[120], [230]], # con questo si specifica la topologia da provare
            "learning_rate": [0.026, 0.029],
            "max_epoch": [13000],
            "momentum": [0.7,  0.8],
            "nesterov": [True],
            "kernel_regularizer": [0.0001, 0.0004],
            "activation": ["sigmoid"],
            "early_stop": [es],
}

grid1 = {
            "n_layer_units": [[120]], # con questo si specifica la topologia da provare
            "learning_rate": [0.00026],
            "max_epoch": [13000],
            "momentum": [0.7],
            "nesterov": [True],
            "kernel_regularizer": [0.0001],
            "activation": ["relu"],
            "early_stop": [es],
}

grid2 = {
            "n_layer_units": [[38]], # con questo si specifica la topologia da provare
            "learning_rate": [0.014, 0.017],
            "max_epoch": [13000],
            "momentum": [0.8],
            "nesterov": [True],
            "kernel_regularizer": [0.0001],
            "activation": ["sigmoid"],
            "early_stop": [es],
}
gs = GridSearchCV(estimator=mlp_r, param_grid = grid1, cv = split, verbose=2)
result = gs.fit(X, Y)
import pandas as pd
df = pd.DataFrame(result)
df
pd.DataFrame(df["fold_results"][0])
kfold = load_data("../CUP/4folds.index")
n_unit = df["hyper_param"][0]["n_layer_units"][0]
lr = df["hyper_param"][0]["learning_rate"]
momentum = df["hyper_param"][0]["momentum"]/2
reg = df["hyper_param"][0]["kernel_regularizer"]
for train_index_fold, val_index_fold, w in zip(kfold["train"], kfold["val"], df["fold_results"][0]["init_weigth"]):
                    
    X_train = dataset[train_index_fold,:-2]
    Y_train = dataset[train_index_fold,-2:]

    X_val = dataset[val_index_fold,:-2]
    Y_val = dataset[val_index_fold,-2:]
    
    num_unit = n_unit
    kernel_regularizer = reg
    model = Mlp()
    model.add(num_unit, activation="sigmoid", input= 20, kernel_initializer = np.sqrt(6)/np.sqrt(num_unit + 20), kernel_regularizer = kernel_regularizer)
    model.add(2, activation="linear", kernel_initializer = np.sqrt(6)/np.sqrt(2 + num_unit), kernel_regularizer = kernel_regularizer)


    model.set_optimizer(
        SGD(
            lr = lr,
            momentum = momentum,
            nesterov = False
        )
    )
    model.weights = w
    es = EarlyStopping(0.00009, 20)

    start_time = time.time()
    model.fit(X_train,
                Y_train, 
                epochs=13000, 
                validation_data = [X_val, Y_val],
                es = es,
                verbose=0) 

    out = model.predict(X_train)
    delta = Y_train - out
    mean_tr_mse.append(np.mean(np.square(delta)))

    outputNet = model.predict(X_val)
    delta = Y_val - outputNet
    mean_vl_mse.append(np.mean(np.square(delta)))

    outputNet = model.predict(X_val)

    out = model.predict(X_train)
    delta = Y_train - out
    mean_tr_mse.append(np.mean(np.square(delta)))

    outputNet = model.predict(X_val)
    delta = Y_val - outputNet
    mean_vl_mse.append(np.mean(np.square(delta)))

print("training: {}+/-{}".format(np.mean(mean_tr_mse), np.std(mean_tr_mse)))
print("validation: {}+/-{}".format(np.mean(mean_vl_mse), np.std(mean_vl_mse)))
pd.DataFrame(df["fold_results"][1])<jupyter_output><empty_output><jupyter_text>Some test on k-folds cup datatest<jupyter_code>TS = np.genfromtxt('../CUP/ML-CUP19-TR_tr_vl.csv',delimiter=',')
kfold = load_kfold("../CUP/4folds.index")
for train_index_fold, val_index_fold in zip(kfold["train"], kfold["val"]):
    X_train = TS[train_index_fold,:-2]
    Y_train = TS[train_index_fold,-2:]
    
    X_val = TS[val_index_fold,:-2]
    Y_val = TS[val_index_fold,-2:]

    model = Mlp()
    model.add(43, activation="sigmoid", input= 20, kernel_initializer = 0.001, kernel_regularizer = 0.001)
    model.add(2, activation="linear", kernel_initializer = 0.001, kernel_regularizer = 0.0001)

    model.set_optimizer(
        SGD(
            lr = 0.019,
            momentum = 0.8,
            nesterov = True
        ))
    # Batch
    start_time = time.time()
    model.fit(X_train,
                Y_train, 
                epochs=10000, 
                #batch_size=31,
                validation_data = [X_val, Y_val],
                verbose=0) 

    outputNet = model.predict(X_val)
    
    print("--- %s seconds ---" % (time.time() - start_time))
    print(model.history["loss_mse"][-1])
    print(model.history["val_loss_mse"][-1])    
n_units = list(range(35, 45+1)) 
lrs = list(np.arange(0.017, 0.021, 0.0004))
momentus = list(np.arange(0.6, 0.9, 0.04))
parameters = []
means = []
devs = []

time_means = []
total_grid_search_iteration = len(n_units)*len(lrs)*len(momentus)
print("Total Grid Search Iteration: {}".format(total_grid_search_iteration))
estimated_time = (total_grid_search_iteration*4*45)/(60*60)
print("Estimated time {}h".format(estimated_time))
i = 0
for units in n_units:
    for lr in lrs:
        for momentum in momentus:
            parameters.append([units, lr, momentum])
            mean = []
            start_time = time.time()
            for train_index_fold, val_index_fold in zip(kfold["train"], kfold["val"]):
                X_train = TS[train_index_fold,:-2]
                Y_train = TS[train_index_fold,-2:]

                X_val = TS[val_index_fold,:-2]
                Y_val = TS[val_index_fold,-2:]

                model = Mlp()
                model.add(units, activation="sigmoid", input= 20, kernel_initializer = 1/np.sqrt(20), kernel_regularizer = 0.001)
                model.add(2, activation="linear", kernel_initializer = 1/np.sqrt(20), kernel_regularizer = 0.0001)

                model.set_optimizer(
                    SGD(
                        lr = lr,
                        momentum = momentum,
                        nesterov = True
                    ))
                # Batch
                start_time = time.time()
                model.fit(X_train,
                            Y_train, 
                            epochs=10, 
                            #batch_size=31,
                            validation_data = [X_val, Y_val],
                            verbose=0) 

                mean.append(model.history["val_loss_mse"][-1])
            means.append(np.mean(mean))
            devs.append(np.std(mean))
            i+=1
            time_means.append((time.time() - start_time))
            remaining_time = (np.mean(time_means)*(total_grid_search_iteration-i))
            print("It: {}/{} - mean: {} - std:{} | Remaining time: {}".format(i,total_grid_search_iteration,means[-1],devs[-1],datetime.timedelta(seconds=remaining_time)))

save_data(parameters, 'parameters.data')
save_data(means, 'means.data')
save_data(devs, 'devs.data')
X_train = TS[:1000,:-2]
Y_train = TS[:1000,-2:]
    
X_val = TS[1000:1250,:-2]
Y_val = TS[1000:1250,-2:]

model = Mlp()
model.add(38, activation="sigmoid", input= 20, kernel_initializer = 1/np.sqrt(20), kernel_regularizer = 0.0001)
model.add(2, activation="linear", kernel_initializer = 1/np.sqrt(100), kernel_regularizer = 0.0001)

model.set_optimizer(
    SGD(
        lr = 0.018200000000000004,
        momentum = 0.8800000000000002,
        nesterov = True
    ))

start_time = time.time()
model.fit(X_train,
            Y_train, 
            epochs=1800, 
            validation_data = [X_val, Y_val],
            verbose=1) 

print("--- %s seconds ---" % (time.time() - start_time))
outputNet = model.predict(X_val)

plt.plot(outputNet[:,-2], outputNet[:,-1], 'ro', markersize=0.3)
plt.ylabel('y2')
plt.xlabel('y1')
plt.tight_layout()
plt.show()

printMSE(outputNet, Y_val, type = "test")
plt.plot(model.history["loss_mse"][500:])
plt.plot(model.history["val_loss_mse"][500:])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.tight_layout()
plt.show()

X_train = TS[:1000,:-2]
Y_train = TS[:1000,-2:]
    
X_val = TS[1000:1250,:-2]
Y_val = TS[1000:1250,-2:]

model = Mlp()
units = 56
model.add(units, activation="sigmoid", input= 20, kernel_initializer = np.sqrt(6)/np.sqrt(20+units), kernel_regularizer = 0.0007108)
model.add(2, activation="linear", kernel_initializer = np.sqrt(6)/np.sqrt(units+2), kernel_regularizer = 0.000108)

model.set_optimizer(
    SGD(
        lr = 0.018200000000000004,
        momentum = 0.8800000000000002,
        nesterov = True
    ))

start_time = time.time()
model.fit(X_train,
            Y_train, 
            epochs=18000, 
            validation_data = [X_val, Y_val],
            verbose=1) 

print("--- %s seconds ---" % (time.time() - start_time))
outputNet = model.predict(X_val)

plt.plot(outputNet[:,-2], outputNet[:,-1], 'ro', markersize=0.3)
plt.ylabel('y2')
plt.xlabel('y1')
plt.tight_layout()
plt.show()

printMSE(outputNet, Y_val, type = "test")
plt.plot(model.history["loss_mse"][500:])
plt.plot(model.history["val_loss_mse"][500:])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.tight_layout()
plt.show()
for train_index_fold, val_index_fold in zip(kfold["train"], kfold["val"]):
    X_train = TS[train_index_fold,:-2]
    Y_train = TS[train_index_fold,-2:]
    
    X_val = TS[val_index_fold,:-2]
    Y_val = TS[val_index_fold,-2:]

    model = Mlp()
    model.add(38, activation="relu", input= 20, kernel_initializer = np.sqrt(6)/np.sqrt(20+62), kernel_regularizer = 0)
    model.add(2, activation="linear", kernel_initializer = np.sqrt(6)/np.sqrt(62+2), kernel_regularizer = 0)

    model.set_optimizer(
        SGD(
            lr = 0.00018200000000000004,
            momentum = 0.8800000000000002,
            nesterov = True
        ))

    start_time = time.time()
    model.fit(X_train,
                Y_train, 
                epochs=6000, 
                validation_data = [X_val, Y_val],
                verbose=1) 

    print("--- %s seconds ---" % (time.time() - start_time))
    print(model.history["loss_mse"][-1])
    print(model.history["val_loss_mse"][-1])<jupyter_output><empty_output><jupyter_text>test with ealry stopping<jupyter_code>X_train = TS[:1000,:-2]
Y_train = TS[:1000,-2:]
    
X_val = TS[1000:1250,:-2]
Y_val = TS[1000:1250,-2:]

model = Mlp()
model.add(52, activation="sigmoid", input= 20, kernel_initializer = 1/np.sqrt(20), kernel_regularizer = 0.0001)
model.add(2, activation="linear", kernel_initializer = 1/np.sqrt(100), kernel_regularizer = 0.0001)

model.set_optimizer(
    SGD(
        lr = 0.018200000000000004,
        momentum = 0.8800000000000002,
        nesterov = True
    ))

start_time = time.time()
model.fit(X_train,
            Y_train, 
            epochs=10000, 
            validation_data = [X_val, Y_val],
            es = 0.0001,
            verbose=1) 

print("--- %s seconds ---" % (time.time() - start_time))
outputNet = model.predict(X_val)

plt.plot(outputNet[:,-2], outputNet[:,-1], 'ro', markersize=0.3)
plt.ylabel('y2')
plt.xlabel('y1')
plt.tight_layout()
plt.show()

printMSE(outputNet, Y_val, type = "test")
plt.plot(model.history["loss_mse"][500:])
plt.plot(model.history["val_loss_mse"][500:])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.tight_layout()
plt.show()<jupyter_output>Epoch: 1 - time: 0.0623 - loss_train: 134.09876838306036 - loss_val: 145.97654379762196
Epoch: 2 - time: 0.0042 - loss_train: 36.033331667180214 - loss_val: 36.97240748090518
Epoch: 3 - time: 0.0070 - loss_train: 50.23105476569823 - loss_val: 49.33762717678182
Epoch: 4 - time: 0.0044 - loss_train: 97.6142077423665 - loss_val: 106.10891571103033
Epoch: 5 - time: 0.0040 - loss_train: 45.623602132894895 - loss_val: 48.63455549724785
Epoch: 6 - time: 0.0043 - loss_train: 11.630462267039297 - loss_val: 11.281772449134184
Epoch: 7 - time: 0.0039 - loss_train: 26.112191369783336 - loss_val: 28.393422986355247
Epoch: 8 - time: 0.0045 - loss_train: 39.27209240495549 - loss_val: 42.882683224697494
Epoch: 9 - time: 0.0039 - loss_train: 35.67434818479178 - loss_val: 38.40378859813673
Epoch: 10 - time: 0.0047 - loss_train: 21.21889947159912 - loss_val: 22.363315854851592
Epoch: 11 - time: 0.0038 - loss_train: 11.512395377667277 - loss_val: 12.039895031732385
Epoch: 12 - time: 0.0038 - loss_train: 1[...]
{'fraction_non_alphanumeric': 0.16030436728691091, 'fraction_numerical': 0.07820193106976149, 'mean_word_length': 2.877045116509668, 'pattern_counts': {'":': 48, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 8, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '573504', 'license': 'MIT', 'n_tokens_mistral': 6602, 'n_tokens_neox': 5641, 'n_words': 1000, 'path': 'notebook/test.ipynb', 'repo_name': 'alessandrocuda/ML_Project_19_20', 'size': '15639'}