<jupyter_start><jupyter_text>Regression<jupyter_code>X, y = datasets.make_regression(n_samples=20000, n_features=20, n_informative=10, noise=100, random_state=1)
X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=1)
M = 1500
learning_rate = 1.0<jupyter_output><empty_output><jupyter_text>early stopping<jupyter_code>gb_early_square = GBRegression(M=M, base_learner=DecisionTreeRegressor(max_depth=1, random_state=1), learning_rate=learning_rate,
                  loss="square", tol=0.02)
gb_early_square.fit(X_train, y_train)
gb_early_huber = GBRegression(M=M, base_learner=DecisionTreeRegressor(max_depth=1, random_state=1), learning_rate=learning_rate,
                  loss="huber", alpha=0.2, tol=0.02)
gb_early_huber.fit(X_train, y_train)<jupyter_output>0.01
0.01
0.01
0.005
early stopping in round 900, best round is 880, M = 880
loss:  103.11501861500744<jupyter_text>----<jupyter_code>gbr = GBRegression(M=M, base_learner=DecisionTreeRegressor(max_depth=1, random_state=1), learning_rate=learning_rate,
                  loss="square")
gbr.fit(X_train, y_train)

gbr_huber = GBRegression(M=M, base_learner=DecisionTreeRegressor(max_depth=1, random_state=1), learning_rate=learning_rate,
                  loss="huber", alpha=0.2)
gbr_huber.fit(X_train, y_train)
gbr_err_train = np.zeros((M+1,))
for i, y_pred in enumerate(gbr.stage_predict(X_train)):
    gbr_err_train[i] = np.sqrt(mean_squared_error(y_pred, y_train))
    
gbr_err_test = np.zeros((M+1,))
for i, y_pred in enumerate(gbr.stage_predict(X_test)):
    gbr_err_test[i] = np.sqrt(mean_squared_error(y_pred, y_test))
    
gbr_huber_err_train = np.zeros((M+1,))
for i, y_pred in enumerate(gbr_huber.stage_predict(X_train)):
    gbr_huber_err_train[i] = np.sqrt(mean_squared_error(y_pred, y_train))
    
gbr_huber_err_test = np.zeros((M+1,))
for i, y_pred in enumerate(gbr_huber.stage_predict(X_test)):
    gbr_huber_err_test[i] = np.sqrt(mean_squared_error(y_pred, y_test))
gbr_early_stopping = GBRegression(M=330, base_learner=DecisionTreeRegressor(max_depth=1, random_state=1), learning_rate=learning_rate,
                  loss="square")
gbr_early_stopping.fit(X_train, y_train)

early_stopping_square = np.sqrt(mean_squared_error(gbr_early_stopping.predict(X_test), y_test))
gbr_huber_early_stopping = GBRegression(M=880, base_learner=DecisionTreeRegressor(max_depth=1, random_state=1), 
                                         learning_rate=learning_rate, loss="huber", alpha=0.2)
gbr_huber_early_stopping.fit(X_train, y_train)

early_stopping_huber = np.sqrt(mean_squared_error(gbr_huber_early_stopping.predict(X_test), y_test))
fig = plt.figure(figsize=(12,8))
ax = fig.add_subplot(111)

ax.plot(np.arange(M+1), gbr_err_train ,label="GBR Square Loss Train Error")
ax.plot(np.arange(M+1), gbr_err_test ,label="GBR Square Loss Test Error",color="orange")

ax.plot([330,330],[85,early_stopping_square],"k--")
plt.plot(330,early_stopping_square, "ko")
plt.text(330,early_stopping_square*1.05,"square loss\nearly stopping point", ha="center", fontsize=14, color="purple")

ax.plot(np.arange(M+1), gbr_huber_err_train ,label="GBR Huber Loss Train Error")
ax.plot(np.arange(M+1), gbr_huber_err_test ,label="GBR Huber Loss Test Error", color="red")

ax.plot([880,880],[85,early_stopping_huber],"k--")
plt.plot(880,early_stopping_huber, "ko")
plt.text(880,early_stopping_huber*1.05,"huber loss\nearly stopping point", ha="center", fontsize=14, color="purple")

ax.legend(loc="best", fontsize=17)
ax.set_ylim([85,200])
ax.set_xlim([-80,1600])
ax.set_xlabel("Num of Rounds", fontsize=17)
ax.set_ylabel("Error Rate", fontsize=17)<jupyter_output><empty_output><jupyter_text>smaller learning rate<jupyter_code>M = 1500
learning_rate = 0.2
gbr = GBRegression(M=M, base_learner=DecisionTreeRegressor(max_depth=1, random_state=1), learning_rate=learning_rate,
                  loss="square")
gbr.fit(X_train, y_train)

gbr_huber = GBRegression(M=M, base_learner=DecisionTreeRegressor(max_depth=1, random_state=1), learning_rate=learning_rate,
                  loss="huber", alpha=0.2)
gbr_huber.fit(X_train, y_train)
gbr_err_train = np.zeros((M+1,))
for i, y_pred in enumerate(gbr.stage_predict(X_train)):
    gbr_err_train[i] = np.sqrt(mean_squared_error(y_pred, y_train))
    
gbr_err_test = np.zeros((M+1,))
for i, y_pred in enumerate(gbr.stage_predict(X_test)):
    gbr_err_test[i] = np.sqrt(mean_squared_error(y_pred, y_test))
    
gbr_huber_err_train = np.zeros((M+1,))
for i, y_pred in enumerate(gbr_huber.stage_predict(X_train)):
    gbr_huber_err_train[i] = np.sqrt(mean_squared_error(y_pred, y_train))
    
gbr_huber_err_test = np.zeros((M+1,))
for i, y_pred in enumerate(gbr_huber.stage_predict(X_test)):
    gbr_huber_err_test[i] = np.sqrt(mean_squared_error(y_pred, y_test))
fig = plt.figure(figsize=(12,8))
ax = fig.add_subplot(111)

ax.plot(np.arange(M+1), gbr_err_train ,label="GBR Square Loss Train Error")
ax.plot(np.arange(M+1), gbr_err_test ,label="GBR Square Loss Test Error",color="orange")
ax.plot(np.arange(M+1), gbr_huber_err_train ,label="GBR Huber Loss Train Error")
ax.plot(np.arange(M+1), gbr_huber_err_test ,label="GBR Huber Loss Test Error", color="red")

ax.legend(loc="best", fontsize=17)
# ax.set_ylim([0,90])
# ax.set_xlim([-80,1600])<jupyter_output><empty_output><jupyter_text>multiple tree depth<jupyter_code>M = 1500
learning_rate = 1.0
gbr_depth1 = GBRegression(M=M, base_learner=DecisionTreeRegressor(max_depth=1, random_state=1), learning_rate=learning_rate,
                  loss="square")
gbr_depth1.fit(X_train, y_train)

gbr_depth2 = GBRegression(M=M, base_learner=DecisionTreeRegressor(max_depth=2, random_state=1), learning_rate=learning_rate,
                  loss="square")
gbr_depth2.fit(X_train, y_train)

gbr_depth4 = GBRegression(M=M, base_learner=DecisionTreeRegressor(max_depth=4, random_state=1), learning_rate=learning_rate,
                  loss="square")
gbr_depth4.fit(X_train, y_train)

gbr_depth8 = GBRegression(M=M, base_learner=DecisionTreeRegressor(max_depth=6, random_state=1), learning_rate=learning_rate,
                  loss="square")
gbr_depth8.fit(X_train, y_train)
gbr_err_depth1 = np.zeros((M+1,))
for i, y_pred in enumerate(gbr_depth1.stage_predict(X_test)):
    gbr_err_depth1[i] = np.sqrt(mean_squared_error(y_pred, y_test))
    
gbr_err_depth2 = np.zeros((M+1,))
for i, y_pred in enumerate(gbr_depth2.stage_predict(X_test)):
    gbr_err_depth2[i] = np.sqrt(mean_squared_error(y_pred, y_test))
    
gbr_err_depth4 = np.zeros((M+1,))
for i, y_pred in enumerate(gbr_depth4.stage_predict(X_test)):
    gbr_err_depth4[i] = np.sqrt(mean_squared_error(y_pred, y_test))
    
gbr_err_depth8 = np.zeros((M+1,))
for i, y_pred in enumerate(gbr_depth8.stage_predict(X_test)):
    gbr_err_depth8[i] = np.sqrt(mean_squared_error(y_pred, y_test))
fig, ax = plt.subplots(1,1, figsize=(12,8))

ax.plot(np.arange(M+1), gbr_err_depth1, label="GBR Tree Depth 1")
ax.plot(np.arange(M+1), gbr_err_depth2, label="GBR Tree Depth 2")
ax.plot(np.arange(M+1), gbr_err_depth4, label="GBR Tree Depth 4")
ax.plot(np.arange(M+1), gbr_err_depth8, label="GBR Tree Depth 8")

ax.legend(loc="best", fontsize=17)
ax.set_ylim([90,200])
ax.set_xlim([-80,1600])<jupyter_output><empty_output><jupyter_text>multiple learning rate & subsampling<jupyter_code>M = 1500
# learning_rate = 1.0
gbr_1 = GBRegression(M=M, base_learner=DecisionTreeRegressor(max_depth=1, random_state=1), learning_rate=1.0,
                  loss="square", subsample=None)
gbr_1.fit(X_train, y_train)

gbr_2 = GBRegression(M=M, base_learner=DecisionTreeRegressor(max_depth=1, random_state=1), learning_rate=0.5,
                  loss="square", subsample=None)
gbr_2.fit(X_train, y_train)

gbr_3 = GBRegression(M=M, base_learner=DecisionTreeRegressor(max_depth=1, random_state=1), learning_rate=1.0,
                  loss="square", subsample=0.8)
gbr_3.fit(X_train, y_train)

gbr_4 = GBRegression(M=M, base_learner=DecisionTreeRegressor(max_depth=1, random_state=1), learning_rate=0.5,
                  loss="square", subsample=0.8)
gbr_4.fit(X_train, y_train)

gbr_5 = GBRegression(M=M, base_learner=DecisionTreeRegressor(max_depth=1, random_state=1), learning_rate=1.0,
                  loss="square", subsample=0.5)
gbr_5.fit(X_train, y_train)

gbr_6 = GBRegression(M=M, base_learner=DecisionTreeRegressor(max_depth=1, random_state=1), learning_rate=0.5,
                  loss="square", subsample=0.5)
gbr_6.fit(X_train, y_train)
gbr_err_1 = np.zeros((M+1,))
for i, y_pred in enumerate(gbr_1.stage_predict(X_test)):
    gbr_err_1[i] = np.sqrt(mean_squared_error(y_pred, y_test))
    
gbr_err_2 = np.zeros((M+1,))
for i, y_pred in enumerate(gbr_2.stage_predict(X_test)):
    gbr_err_2[i] = np.sqrt(mean_squared_error(y_pred, y_test))
    
gbr_err_3 = np.zeros((M+1,))
for i, y_pred in enumerate(gbr_3.stage_predict(X_test)):
    gbr_err_3[i] = np.sqrt(mean_squared_error(y_pred, y_test))
    
gbr_err_4 = np.zeros((M+1,))
for i, y_pred in enumerate(gbr_4.stage_predict(X_test)):
    gbr_err_4[i] = np.sqrt(mean_squared_error(y_pred, y_test))
    
gbr_err_5 = np.zeros((M+1,))
for i, y_pred in enumerate(gbr_5.stage_predict(X_test)):
    gbr_err_5[i] = np.sqrt(mean_squared_error(y_pred, y_test))
    
gbr_err_6 = np.zeros((M+1,))
for i, y_pred in enumerate(gbr_6.stage_predict(X_test)):
    gbr_err_6[i] = np.sqrt(mean_squared_error(y_pred, y_test))
fig, ax = plt.subplots(1,1,figsize=(12,8))

ax.plot(np.arange(M+1), gbr_err_1, label="GBR learning rate=1.0, subsample=None", color="orange")
ax.plot(np.arange(M+1), gbr_err_2, label="GBR learning rate=0.5, subsample=None")
ax.plot(np.arange(M+1), gbr_err_3, label="GBR learning rate=1.0, subsample=0.8")
ax.plot(np.arange(M+1), gbr_err_4, label="GBR learning rate=0.5, subsample=0.8")
ax.plot(np.arange(M+1), gbr_err_5, color="#FF4500", label="GBR learning rate=1.0, subsample=0.5")
ax.plot(np.arange(M+1), gbr_err_5, label="GBR learning rate=0.5, subsample=0.5")

ax.legend(loc="best", fontsize=17)
# ax.set_ylim([0,90])
# ax.set_xlim([-80,1600])<jupyter_output><empty_output><jupyter_text>Classification<jupyter_code>from AdaBoost_implementation import AdaBoost<jupyter_output><empty_output><jupyter_text>Normal Data<jupyter_code>X,y = datasets.make_hastie_10_2(n_samples=20000, random_state=42)
X_train,X_test,y_train,y_test = train_test_split(X,y,random_state=1)
pca = PCA(n_components=2)
X_pca = pca.fit_transform(X)
X_pca = pd.DataFrame(X_pca)
X_pca.columns=["pca_a","pca_b"]
X_pca["y"] = y
sns.lmplot("pca_a","pca_b",data=X_pca[:2000], hue="y", fit_reg=False, markers=["o","x"],size=8,aspect=1.5,legend=False)
plt.legend(fontsize=20,bbox_to_anchor=(0.98, 0.6),edgecolor ='r')   
plt.xlabel("axis_1",fontsize=17)
plt.ylabel("axis_2",fontsize=17)
M = 1500
learning_rate = 1.0
ada_discrete = AdaBoost(M=M, clf=DecisionTreeClassifier(max_depth=1,random_state=1),learning_rate=learning_rate,
                        method="discrete",tol=None)
ada_discrete.fit(X_train, y_train)
ada_real = AdaBoost(M=M, clf=DecisionTreeClassifier(max_depth=1, random_state=1),learning_rate=learning_rate,
                    method="real",tol=None)
ada_real.fit(X_train, y_train)
ada_discrete_err = np.zeros((M,))
for i, y_pred in enumerate(ada_discrete.stage_predict(X_test)):
    ada_discrete_err[i] = zero_one_loss(y_pred, y_test)
    
ada_real_err = np.zeros((M,))
for i, y_pred in enumerate(ada_real.stage_predict(X_test)):
    ada_real_err[i] = zero_one_loss(y_pred, y_test)
gbc = GBClassification(M=M, base_learner=DecisionTreeRegressor(max_depth=1, random_state=1), 
                       learning_rate=learning_rate, loss="logistic")
gbc.fit(X_train, y_train)
gbc_huber = GBClassification(M=M, base_learner=DecisionTreeRegressor(max_depth=1, random_state=1), 
                       learning_rate=learning_rate, loss="modified_huber")
gbc_huber.fit(X_train, y_train)
gbc_error = np.zeros((M+1,))
for i, y_pred in enumerate(gbc.stage_predict(X_test)):
    gbc_error[i] = zero_one_loss(y_pred, y_test)
    
gbc_huber_error = np.zeros((M+1,))
for i, y_pred in enumerate(gbc_huber.stage_predict(X_test)):
    gbc_huber_error[i] = zero_one_loss(y_pred, y_test)
fig = plt.figure(figsize=(12,8))
ax = fig.add_subplot(111)

ax.plot(np.arange(M), ada_discrete_err ,label="Discrete AdaBoost Test Error")
ax.plot(np.arange(M), ada_real_err ,label="Real AdaBoost Test Error")
ax.plot(np.arange(M+1), gbc_error ,label="Gradient Boosting Test Error")
ax.plot(np.arange(M+1), gbc_huber_error ,label="Gradient Boosting Test Error - huber loss")

ax.legend(loc="best", fontsize=17)
ax.set_ylim([0,0.5])
ax.set_xlim([-80,1600])<jupyter_output><empty_output><jupyter_text>Noisy Data<jupyter_code>M = 1500
learning_rate = 1.0
X, y = datasets.make_classification(n_samples=20000, n_features=10, n_informative=4, flip_y=0.3, 
                                    n_clusters_per_class=1, n_classes=2, random_state=1)
y[y==0] = -1
X_train, X_test, y_train, y_test = train_test_split(X,y,random_state=1)
pca = PCA(n_components=2)
X_pca = pca.fit_transform(X)
X_pca = pd.DataFrame(X_pca)
X_pca.columns=["pca_a","pca_b"]
X_pca["y"] = y
sns.lmplot("pca_a","pca_b",data=X_pca[:2000], hue="y", fit_reg=False, markers=["o","x"],size=8,aspect=1.5,legend=False)
plt.legend(fontsize=20,bbox_to_anchor=(0.98, 0.6),edgecolor ='r')   
plt.xlabel("axis_1",fontsize=17)
plt.ylabel("axis_2",fontsize=17)
ada_discrete = AdaBoost(M=M, clf=DecisionTreeClassifier(max_depth=1,random_state=1),learning_rate=learning_rate,
                        method="discrete",tol=None)
ada_discrete.fit(X_train, y_train)
ada_real = AdaBoost(M=M, clf=DecisionTreeClassifier(max_depth=1, random_state=1),learning_rate=learning_rate,
                    method="real",tol=None)
ada_real.fit(X_train, y_train)
ada_discrete_err = np.zeros((M,))
for i, y_pred in enumerate(ada_discrete.stage_predict(X_test)):
    ada_discrete_err[i] = zero_one_loss(y_pred, y_test)
    
ada_real_err = np.zeros((M,))
for i, y_pred in enumerate(ada_real.stage_predict(X_test)):
    ada_real_err[i] = zero_one_loss(y_pred, y_test)
gbc = GBClassification(M=M, base_learner=DecisionTreeRegressor(max_depth=1, random_state=1), 
                       learning_rate=learning_rate, method="classification", loss="logistic")
gbc.fit(X_train, y_train)
gbc_huber = GBClassification(M=M, base_learner=DecisionTreeRegressor(max_depth=1, random_state=1), 
                       learning_rate=learning_rate, method="classification", loss="modified_huber")
gbc_huber.fit(X_train, y_train)
gbc_error = np.zeros((M+1,))
for i, y_pred in enumerate(gbc.stage_predict(X_test)):
    gbc_error[i] = zero_one_loss(y_pred, y_test)
    
gbc_huber_error = np.zeros((M+1,))
for i, y_pred in enumerate(gbc_huber.stage_predict(X_test)):
    gbc_huber_error[i] = zero_one_loss(y_pred, y_test)
fig = plt.figure(figsize=(12,8))
ax = fig.add_subplot(111)

ax.plot(np.arange(M), ada_discrete_err ,label="Discrete AdaBoost Test Error")
ax.plot(np.arange(M), ada_real_err ,label="Real AdaBoost Test Error")
ax.plot(np.arange(M+1), gbc_error ,label="Gradient Boosting Test Error")
ax.plot(np.arange(M+1), gbc_huber_error ,label="Gradient Boosting Test Error - huber loss")

ax.legend(loc="best", fontsize=17)
# ax.set_ylim([0,0.5])
ax.set_xlim([-80,1600])<jupyter_output><empty_output><jupyter_text>smaller learing rate<jupyter_code>M = 3000
learning_rate = 0.1
ada_discrete = AdaBoost(M=M, clf=DecisionTreeClassifier(max_depth=1,random_state=1),learning_rate=learning_rate,
                        method="discrete",tol=None)
ada_discrete.fit(X_train, y_train)
ada_real = AdaBoost(M=M, clf=DecisionTreeClassifier(max_depth=1, random_state=1),learning_rate=learning_rate,
                    method="real",tol=None)
ada_real.fit(X_train, y_train)
ada_discrete_err = np.zeros((M,))
for i, y_pred in enumerate(ada_discrete.stage_predict(X_test)):
    ada_discrete_err[i] = zero_one_loss(y_pred, y_test)
    
ada_real_err = np.zeros((M,))
for i, y_pred in enumerate(ada_real.stage_predict(X_test)):
    ada_real_err[i] = zero_one_loss(y_pred, y_test)
gbc = GBClassification(M=M, base_learner=DecisionTreeRegressor(max_depth=1, random_state=1), 
                       learning_rate=learning_rate, method="classification", loss="logistic",
                      tol=None, subsample=None)
gbc.fit(X_train, y_train)
gbc_huber = GBClassification(M=M, base_learner=DecisionTreeRegressor(max_depth=1, random_state=1), 
                           learning_rate=learning_rate, method="classification", loss="modified_huber",
                            tol=None, subsample=None)
gbc_huber.fit(X_train, y_train)
gbc_error = np.zeros((M+1,))
for i, y_pred in enumerate(gbc.stage_predict(X_test)):
    gbc_error[i] = zero_one_loss(y_pred, y_test)
    
gbc_huber_error = np.zeros((M+1,))
for i, y_pred in enumerate(gbc_huber.stage_predict(X_test)):
    gbc_huber_error[i] = zero_one_loss(y_pred, y_test)
fig = plt.figure(figsize=(12,8))
ax = fig.add_subplot(111)

ax.plot(np.arange(M), ada_discrete_err ,label="Discrete AdaBoost Test Error")
ax.plot(np.arange(M), ada_real_err ,label="Real AdaBoost Test Error")
ax.plot(np.arange(M+1), gbc_error ,label="Gradient Boosting Test Error")
ax.plot(np.arange(M+1), gbc_huber_error ,label="Gradient Boosting Test Error - huber loss")

ax.legend(loc="best", fontsize=17)
# ax.set_ylim([0,0.5])
# ax.set_xlim([-80,1600])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14565518835220706, 'fraction_numerical': 0.03252831060781142, 'mean_word_length': 6.288, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '82575', 'license': 'MIT', 'n_tokens_mistral': 7528, 'n_tokens_neox': 7119, 'n_words': 957, 'path': 'Gradient Boosting/GradientBoosting_plot-Article.ipynb', 'repo_name': 'massquantity/ML-algorithms-implemetation', 'size': '17308'}