<jupyter_start><jupyter_text>instability with large learning rate<jupyter_code>np.random.seed(42)
param = {
            "n_layer_units": [50], # con questo si specifica la topologia da provare
            "learning_rate": 0.3,
            #"momentum": 0.9,
            "nesterov": False,
            #"kernel_regularizer": 0.0001,
            "activation": "sigmoid",
            #"early_stop": EarlyStopping(0.00009, 20),
            "max_epoch": 3000
}
model = MLPRegressor(X_train.shape[1], Y_train.shape[1], random_state = 42, verbose = 0, **param)
model.fit(X_train, Y_train, X_val, Y_val)

outputNet = model.predict(X_train)
printMSE(outputNet, Y_train, type = "train")

outputNet = model.predict(X_val)
printMSE(outputNet, Y_val, type = "val")

plotCup(model.get_history())
plotCupCutUnder(model.get_history(), name="cup_instability")
plotCupCut(model.get_history(), name="cup_instability")<jupyter_output>MSE train: 4.19413701654776 
MSE val: 5.764902275149154<jupyter_text>no momentum<jupyter_code>np.random.seed(42)
param = {
            "n_layer_units": [50], # con questo si specifica la topologia da provare
            "learning_rate": 0.01,
            #"momentum": 0.9,
            "nesterov": False,
            #"kernel_regularizer": 0.0001,
            "activation": "sigmoid",
            #"early_stop": EarlyStopping(0.00009, 20),
            "max_epoch": 100
}
model = MLPRegressor(X_train.shape[1], Y_train.shape[1], random_state = 42, verbose = 0, **param)
model.fit(X_train, Y_train, X_val, Y_val)

outputNet = model.predict(X_train)
printMSE(outputNet, Y_train, type = "train")

outputNet = model.predict(X_val)
printMSE(outputNet, Y_val, type = "val")

#plotCup(model.get_history())
plotCupCutUnder(model.get_history(), name="cup_no_momentum")
#plotCupCut(model.get_history())<jupyter_output>The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.
The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.<jupyter_text>Momentum<jupyter_code>np.random.seed(42)

param = {
            "n_layer_units": [50], # con questo si specifica la topologia da provare
            "learning_rate": 0.01,
            "momentum": 0.9,
            "nesterov": False,
            #"kernel_regularizer": 0.0001,
            "activation": "sigmoid",
            #"early_stop": EarlyStopping(0.00009, 20),
            "max_epoch": 100
}
model = MLPRegressor(X_train.shape[1], Y_train.shape[1], random_state = 42, verbose = 0, **param)
model.fit(X_train, Y_train, X_val, Y_val)

outputNet = model.predict(X_train)
printMSE(outputNet, Y_train, type = "train")

outputNet = model.predict(X_val)
printMSE(outputNet, Y_val, type = "val")

#plotCup(model.get_history())
plotCupCutUnder(model.get_history())
#plotCupCut(model.get_history())<jupyter_output>MSE train: 2.5101098433928493 
MSE val: 2.6121749197549717<jupyter_text>Nesterov<jupyter_code>np.random.seed(42)

param = {
            "n_layer_units": [50], # con questo si specifica la topologia da provare
            "learning_rate": 0.01,
            "momentum": 0.9,
            "nesterov": True,
            #"kernel_regularizer": 0.0001,
            "activation": "sigmoid",
            #"early_stop": EarlyStopping(0.00009, 20),
            "max_epoch": 100
}
model = MLPRegressor(X_train.shape[1], Y_train.shape[1], random_state = 42, verbose = 0, **param)
model.fit(X_train, Y_train, X_val, Y_val)

outputNet = model.predict(X_train)
printMSE(outputNet, Y_train, type = "train")

outputNet = model.predict(X_val)
printMSE(outputNet, Y_val, type = "val")

#plotCup(model.get_history())
plotCupCutUnder(model.get_history(), name="cup_with_nesterov")
#plotCupCut(model.get_history())<jupyter_output>The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.
The PostScript backend does not support transparency; partially transparent artists will be rendered opaque.<jupyter_text>Super-accelerated Nesterov<jupyter_code>param = {
            "n_layer_units": [50], # con questo si specifica la topologia da provare
            "learning_rate": 0.01,
            "momentum": 0.9,
            "nesterov": True,
            "sigma": 4,
            #"kernel_regularizer": 0.0001,
            "activation": "sigmoid",
            #"early_stop": EarlyStopping(0.00009, 20),
            "max_epoch": 25000
}
model = MLPRegressor(X_train.shape[1], Y_train.shape[1], random_state = 42, verbose = 0, **param)
model.fit(X_train, Y_train, X_val, Y_val)

outputNet = model.predict(X_train)
printMSE(outputNet, Y_train, type = "train")

outputNet = model.predict(X_val)
printMSE(outputNet, Y_val, type = "val")

plotCup(model.get_history())
plotCupCutUnder(model.get_history())
plotCupCut(model.get_history())
param = {
            "n_layer_units": [50], # con questo si specifica la topologia da provare
            "learning_rate": 0.01,
            "momentum": 0.9,
            "nesterov": True,
            "sigma": 6,
            #"kernel_regularizer": 0.0001,
            "activation": "sigmoid",
            #"early_stop": EarlyStopping(0.00009, 20),
            "max_epoch": 25000
}
model = MLPRegressor(X_train.shape[1], Y_train.shape[1], random_state = 42, verbose = 0, **param)
model.fit(X_train, Y_train, X_val, Y_val)

outputNet = model.predict(X_train)
printMSE(outputNet, Y_train, type = "train")

outputNet = model.predict(X_val)
printMSE(outputNet, Y_val, type = "val")

plotCup(model.get_history())
plotCupCutUnder(model.get_history())
plotCupCut(model.get_history())<jupyter_output>MSE train: 0.7521502017197524 
MSE val: 0.9687060063909088<jupyter_text>MONK<jupyter_code>print("Load Monk DataSet")
X_train, Y_train = load_monk("1", "train")
X_test, Y_test = load_monk("1", "test")
seed = 82
epoch = 1500
np.random.seed(seed=seed)
print("Build the model")
model0 = Mlp()
model0.add(4, input= 17, kernel_initializer = 0.003, kernel_regularizer = 0.001)
model0.add(1, kernel_initializer = 0.003, kernel_regularizer = 0.001)

model0.set_optimizer( SGD( lr = 0.83, momentum = 0.9, nesterov = True))
# Batch
print("Start to Fit:")
model0.fit(X_train, Y_train, epochs=epoch, validation_data = [X_test, Y_test], verbose=0) 

np.random.seed(seed=seed)
print("Build the model")
model1 = Mlp()
model1.add(4, input= 17, kernel_initializer = 0.003, kernel_regularizer = 0.001)
model1.add(1, kernel_initializer = 0.003, kernel_regularizer = 0.001)

model1.set_optimizer( SGD( lr = 0.83, momentum = 0.9, nesterov = True, sigma = 8))
# Batch
print("Start to Fit:")
model1.fit(X_train, Y_train, epochs=epoch, validation_data = [X_test, Y_test], verbose=0) 

np.random.seed(seed=seed)
print("Build the model")
model2 = Mlp()
model2.add(4, input= 17, kernel_initializer = 0.003, kernel_regularizer = 0.001)
model2.add(1, kernel_initializer = 0.003, kernel_regularizer = 0.001)

model2.set_optimizer( SGD( lr = 0.83, momentum = 0.9))
# Batch
print("Start to Fit:")
model2.fit(X_train, Y_train, epochs=epoch, validation_data = [X_test, Y_test], verbose=0) 

np.random.seed(seed=seed)
print("Build the model")
model3 = Mlp()
model3.add(4, input= 17, kernel_initializer = 0.003, kernel_regularizer = 0.001)
model3.add(1, kernel_initializer = 0.003, kernel_regularizer = 0.001)

model3.set_optimizer( SGD( lr = 0.83))
# Batch
print("Start to Fit:")
model3.fit(X_train, Y_train, epochs=epoch, validation_data = [X_test, Y_test], verbose=0) 


plt.plot(model3.history["loss_mse"], linestyle=':')
plt.plot(model2.history["loss_mse"], linestyle='-.')
plt.plot(model0.history["loss_mse"])
plt.plot(model1.history["loss_mse"], linestyle='--')
plt.title('MSE')
plt.ylabel('MSE')
plt.xlabel('Epoch')
plt.grid()
plt.legend(['No Momentum','Momentum', 'Nesterov', 'Acc. Nesterov'], loc='upper right', fontsize='large')
plt.tight_layout()
plt.savefig("momentum_img/monk1_4_momentum.eps")
plt.show()<jupyter_output>Build the model
Start to Fit:
Build the model
Start to Fit:
Build the model
Start to Fit:
Build the model
Start to Fit:
{'fraction_non_alphanumeric': 0.14150355317292107, 'fraction_numerical': 0.052113202842538335, 'mean_word_length': 4.31610337972167, 'pattern_counts': {'":': 50, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '107021', 'license': 'MIT', 'n_tokens_mistral': 3225, 'n_tokens_neox': 2895, 'n_words': 604, 'path': 'cup_project/notebook/momentums_comparison.ipynb', 'repo_name': 'alessandrocuda/ML_Project_19_20', 'size': '8021'}