<jupyter_start><jupyter_text>LinearRegression<jupyter_code>from sklearn.linear_model import LinearRegression

lin_reg = LinearRegression()
lin_reg.fit(X_train,y_train)
print(lin_reg.intercept_, lin_reg.coef_, "\n")
lin_test_y = lin_reg.predict(X_test)
lin_train_y = lin_reg.predict(X_train)
# clear plot if any occurs
plt.clf()
# plot initial points
plt.plot(X_test, y_test,'.')
# plot linear regression result
plt.plot(X_test, lin_test_y)
# clear plot if any occurs
plt.clf()
# plot initial points
plt.plot(X_train, y_train,'.')
# plot linear regression result
plt.plot(X_train, lin_train_y)<jupyter_output><empty_output><jupyter_text>KNN<jupyter_code>from sklearn.neighbors import KNeighborsRegressor

# k = 3 
knn_3_reg = KNeighborsRegressor(n_neighbors=3)
knn_3_reg.fit(X_train,y_train)
knn_3_test_y = knn_3_reg.predict(X_test)
knn_3_train_y = knn_3_reg.predict(X_train)
# clear plot if any occurs
plt.clf()
# plot initial points
plt.plot(X_test, y_test,'.')
# plot KNN result
plt.plot(X_test, knn_3_test_y, 'r.')
# k = 5 
knn_5_reg = KNeighborsRegressor(n_neighbors=5)
knn_5_reg.fit(X_train,y_train)
knn_5_test_y = knn_5_reg.predict(X_test)
# clear plot if any occurs
plt.clf()
# plot initial points
plt.plot(X_test, y_test,'.')
# plot KNN result
plt.plot(X_test, knn_5_test_y, 'r.')<jupyter_output><empty_output><jupyter_text>Polynomials<jupyter_code>from sklearn.preprocessing import PolynomialFeatures
def plot_polynomial(X, y, predY):
    # clear plot
    plt.clf()
    # plot initial points
    plt.plot(X, y,'.')
    # plot predictions
    plt.plot(X, predY,'r.')<jupyter_output><empty_output><jupyter_text>2ND GRADE POLYNOMIAL<jupyter_code># 2
poly_feature_2 = PolynomialFeatures(degree=2, include_bias=False)
X_poly2 = poly_feature_2.fit_transform(X_train)

poly_2_reg = LinearRegression()
poly_2_reg.fit(X_poly2, y_train)
print("Coefficients: ", poly_2_reg.coef_, "\n", "Intercept: ", poly_2_reg.intercept_,"\n")
lin2_predY = poly_2_reg.predict(poly_feature_2.fit_transform(X_test))
print(poly_2_reg.coef_[1], " x**2 + ", poly_2_reg.coef_[0], " x + ", poly_2_reg.intercept_)
# print(poly_2_reg.coef_[0][1] * 2**2 + poly_2_reg.coef_[0][0] * 2 + poly_2_reg.intercept_[0])
# plot test set
plot_polynomial(X_test, y_test, lin2_predY)
# plot all points
plot_polynomial(X, y, poly_2_reg.predict(poly_feature_2.fit_transform(X)))<jupyter_output><empty_output><jupyter_text>3RD GRADE POLYNOMIAL<jupyter_code>poly_feature_3 = PolynomialFeatures(degree=3, include_bias=False)
X_poly3 = poly_feature_3.fit_transform(X_train)

poly_3_reg = LinearRegression()
poly_3_reg.fit(X_poly3, y_train)
print("Coefficients: ", poly_3_reg.coef_, "\n", "Intercept: ", poly_3_reg.intercept_,"\n")
lin3_predY = poly_3_reg.predict(poly_feature_3.fit_transform(X_test))
print(poly_3_reg.coef_[2], "x**3 +", poly_3_reg.coef_[1], "x**2 +", poly_3_reg.coef_[0], "x + ", poly_3_reg.intercept_)
# plot test set
plot_polynomial(X_test, y_test, lin3_predY)
# plot all points
plot_polynomial(X, y, poly_3_reg.predict(poly_feature_3.fit_transform(X)))<jupyter_output><empty_output><jupyter_text>4TH GRADE POLYNOMIAL<jupyter_code>poly_feature_4 = PolynomialFeatures(degree=4, include_bias=False)
X_poly4 = poly_feature_4.fit_transform(X_train)

poly_4_reg = LinearRegression()
poly_4_reg.fit(X_poly4, y_train)
print("Coefficients: ", poly_4_reg.coef_, "\n", "Intercept: ", poly_4_reg.intercept_,"\n")
lin4_predY = poly_4_reg.predict(poly_feature_4.fit_transform(X_test))
print(poly_4_reg.coef_[3], "x**4 +", poly_4_reg.coef_[2], "x**3 +", poly_4_reg.coef_[1], "x**2 +", poly_4_reg.coef_[0], "x + ", poly_4_reg.intercept_)
# plot test set
plot_polynomial(X_test, y_test, lin4_predY)
# plot all points
plot_polynomial(X, y, poly_4_reg.predict(poly_feature_4.fit_transform(X)))<jupyter_output><empty_output><jupyter_text>5TH GRADE POLYNOMIAL<jupyter_code>poly_feature_5 = PolynomialFeatures(degree=5, include_bias=False)
X_poly5 = poly_feature_5.fit_transform(X_train)

poly_5_reg = LinearRegression()
poly_5_reg.fit(X_poly5, y_train)
print("Coefficients: ", poly_5_reg.coef_, "\n", "Intercept: ", poly_5_reg.intercept_,"\n")
lin5_predY = poly_5_reg.predict(poly_feature_5.fit_transform(X_test))
print(poly_5_reg.coef_[4], "x**5 +", poly_5_reg.coef_[3], "x**4 +", poly_5_reg.coef_[2], "x**3 +", poly_5_reg.coef_[1], "x**2 +", poly_5_reg.coef_[0], "x + ", poly_5_reg.intercept_)
# plot test set
plot_polynomial(X_test, y_test, lin5_predY)
# plot all points
plot_polynomial(X, y, poly_5_reg.predict(poly_feature_5.fit_transform(X)))<jupyter_output><empty_output><jupyter_text>Mean Squared Error (MSE)<jupyter_code>from sklearn.metrics import mean_squared_error<jupyter_output><empty_output><jupyter_text>lin_reg MSE<jupyter_code># lin_reg MSE
lin_reg_pred_test = lin_reg.predict(X_test)
lin_reg_pred_train = lin_reg.predict(X_train)
lin_test_mse = mean_squared_error(y_test, lin_reg_pred_test)
lin_train_mse = mean_squared_error(y_train, lin_reg_pred_train)
print("lin_test_mse: ", lin_test_mse)
print("lin_train_mse: ", lin_train_mse)<jupyter_output>lin_test_mse:  219.47838931188318
lin_train_mse:  242.34054882516992<jupyter_text>KNN_reg MSE<jupyter_code>knn_3_reg_pred_test = knn_3_reg.predict(X_test)
knn_3_reg_pred_train = knn_3_reg.predict(X_train)
knn_3_test_mse = mean_squared_error(y_test, knn_3_reg_pred_test)
knn_3_train_mse = mean_squared_error(y_train, knn_3_reg_pred_train)
print("knn_3_test_mse: ", knn_3_test_mse)
print("knn_3_train_mse: ", knn_3_train_mse)
knn_5_reg_pred_test = knn_5_reg.predict(X_test)
knn_5_reg_pred_train = knn_5_reg.predict(X_train)
knn_5_test_mse = mean_squared_error(y_test, knn_5_reg_pred_test)
knn_5_train_mse = mean_squared_error(y_train, knn_5_reg_pred_train)
print("knn_5_test_mse: ", knn_5_test_mse)
print("knn_5_train_mse: ", knn_5_train_mse)<jupyter_output>knn_5_test_mse:  50.24383743403386
knn_5_train_mse:  46.106491794808825<jupyter_text>Polynomial MSE<jupyter_code>poly_2_reg_pred_test = poly_2_reg.predict(poly_feature_2.fit_transform(X_test))
poly_2_reg_pred_train = poly_2_reg.predict(poly_feature_2.fit_transform(X_train))
poly_2_test_mse = mean_squared_error(y_test, poly_2_reg_pred_test)
poly_2_train_mse = mean_squared_error(y_train, poly_2_reg_pred_train)
print("poly_2_test_mse: ", poly_2_test_mse)
print("poly_2_train_mse: ", poly_2_train_mse)
poly_3_reg_pred_test = poly_3_reg.predict(poly_feature_3.fit_transform(X_test))
poly_3_reg_pred_train = poly_3_reg.predict(poly_feature_3.fit_transform(X_train))
poly_3_test_mse = mean_squared_error(y_test, poly_3_reg_pred_test)
poly_3_train_mse = mean_squared_error(y_train, poly_3_reg_pred_train)
print("poly_3_test_mse: ", poly_3_test_mse)
print("poly_3_train_mse: ", poly_3_train_mse)
poly_4_reg_pred_test = poly_4_reg.predict(poly_feature_4.fit_transform(X_test))
poly_4_reg_pred_train = poly_4_reg.predict(poly_feature_4.fit_transform(X_train))
poly_4_test_mse = mean_squared_error(y_test, poly_4_reg_pred_test)
poly_4_train_mse = mean_squared_error(y_train, poly_4_reg_pred_train)
print("poly_4_test_mse: ", poly_4_test_mse)
print("poly_4_train_mse: ", poly_4_train_mse)
poly_5_reg_pred_test = poly_5_reg.predict(poly_feature_5.fit_transform(X_test))
poly_5_reg_pred_train = poly_5_reg.predict(poly_feature_5.fit_transform(X_train))
poly_5_test_mse = mean_squared_error(y_test, poly_5_reg_pred_test)
poly_5_train_mse = mean_squared_error(y_train, poly_5_reg_pred_train)
print("poly_5_test_mse: ", poly_5_test_mse)
print("poly_5_train_mse: ", poly_5_train_mse)<jupyter_output>poly_5_test_mse:  39.03481336258604
poly_5_train_mse:  58.02378001008437<jupyter_text>MSE DataFrame<jupyter_code>mse_df = pd.DataFrame({
        "regressors": ['lin_reg', 'knn_3_reg', 'knn_5_reg', 'poly_2_reg', 'poly_3_reg', 'poly_4_reg', 'poly_5_reg'],
        "train_mse": [lin_train_mse, knn_3_train_mse, knn_5_train_mse, poly_2_train_mse, poly_3_train_mse, poly_4_train_mse, poly_5_train_mse],
        "test_mse": [lin_test_mse, knn_3_test_mse, knn_5_test_mse, poly_2_test_mse, poly_3_test_mse, poly_4_test_mse, poly_5_test_mse]
    }).set_index("regressors")

mse_df
print("Columns:", mse_df.columns)
print("Index: ", mse_df.index)
# save mse_df as pickle
mse_df.to_pickle('mse.pkl')<jupyter_output><empty_output><jupyter_text>Regression objects<jupyter_code>reg_objects = [(lin_reg, None), (knn_3_reg, None), (knn_5_reg, None), (poly_2_reg, poly_feature_2), 
                 (poly_3_reg, poly_feature_3), (poly_4_reg, poly_feature_4), (poly_5_reg, poly_feature_5)]
print(reg_objects)

# save reg_objects as pickle
filename = "reg.pkl"
with open(filename, 'wb') as file:
        pickle.dump(reg_objects, file, pickle.HIGHEST_PROTOCOL)<jupyter_output>[(LinearRegression(), None), (KNeighborsRegressor(n_neighbors=3), None), (KNeighborsRegressor(), None), (LinearRegression(), PolynomialFeatures(include_bias=False)), (LinearRegression(), PolynomialFeatures(degree=3, include_bias=False)), (LinearRegression(), PolynomialFeatures(degree=4, include_bias=False)), (LinearRegression(), PolynomialFeatures(degree=5, include_bias=False))]<jupyter_text>Check saved Pickles contents<jupyter_code># check if pickles' contents are saved correctly
print("acc_list\n", pd.read_pickle("mse.pkl"), "\n")
print("reg_objects\n", pd.read_pickle("reg.pkl"))<jupyter_output>acc_list
              train_mse    test_mse
regressors                        
lin_reg     242.340549  219.478389
knn_3_reg    38.901071   48.559748
knn_5_reg    46.106492   50.243837
poly_2_reg   90.584374   77.035997
poly_3_reg   70.646404   46.978378
poly_4_reg   58.027983   38.932580
poly_5_reg   58.023780   39.034813 

reg_objects
 [(LinearRegression(), None), (KNeighborsRegressor(n_neighbors=3), None), (KNeighborsRegressor(), None), (LinearRegression(), PolynomialFeatures(include_bias=False)), (LinearRegression(), PolynomialFeatures(degree=3, include_bias=False)), (LinearRegression(), PolynomialFeatures(degree=4, include_bias=False)), (LinearRegression(), PolynomialFeatures(degree=5, include_bias=False))]
{'fraction_non_alphanumeric': 0.12220881110440555, 'fraction_numerical': 0.047575940454636895, 'mean_word_length': 9.043434343434344, 'pattern_counts': {'":': 3, '<': 52, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '76306', 'license': 'MIT', 'n_tokens_mistral': 4472, 'n_tokens_neox': 4246, 'n_words': 531, 'path': 'lab3/lab3.ipynb', 'repo_name': 'YgLK/ML', 'size': '9942'}