<jupyter_start><jupyter_text>Import Data<jupyter_code>df_train = pd.read_csv(r"data\train.csv")
df_test = pd.read_csv(r"data\test.csv")
df_train.head()
X_train = df_train.drop(columns="Rating")
y_train = df_train["Rating"]

X_test = df_test.drop(columns="Rating")
y_test = df_test["Rating"]
print(f"Number of training samples: {X_train.shape[0]}")
print(f"Number of test samples: {X_test.shape[0]}")<jupyter_output>Number of training samples: 1436
Number of test samples: 359<jupyter_text>Train Benchmark Model (Linear Regression)<jupyter_code>from sklearn.linear_model import LinearRegression
from sklearn.metrics import r2_score, mean_squared_error
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import cross_val_predict

estimator = Pipeline([
                    ("scaler", StandardScaler()), 
                    ("regression", LinearRegression())
                    ])

# perform cross-validation
predictions = cross_val_predict(estimator, X_train, y_train, cv=5)

# get scores training
r2_score_train = r2_score(y_train, predictions)
mse_train = mean_squared_error(y_train, predictions)

results_dict[estimator_name] = (r2_score_train, mse_train)
print(f"MSE = {mse_train} | R-Squared = {r2_score_train}")<jupyter_output>MSE = 1.4267370043236246e+26 | R-Squared = -6.267064373130277e+26<jupyter_text>The R-Squared score is negative! That means that the normal linear regression model fits worse than a horizontal line.  Train other Regression Models1. Ridge Regression.2. Decision Tree Regressor.3. Random Forest Regressor.4. Gradient Boosting Regressor.<jupyter_code>from sklearn.linear_model import Ridge
from sklearn.preprocessing import PolynomialFeatures
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import GradientBoostingRegressor
estimator_ridge = Pipeline([
                            ("scaler", StandardScaler()), 
                            ("regression", Ridge())
                            ])

estimator_decision_tree = Pipeline([
                                    ("scaler", StandardScaler()), 
                                    ("regression", DecisionTreeRegressor())
                                    ])

estimator_random_forest = Pipeline([
                                    ("scaler", StandardScaler()), 
                                    ("regression", RandomForestRegressor())
                                    ])

estimator_gradient_boosting = Pipeline([
                                        ("scaler", StandardScaler()), 
                                        ("regression", GradientBoostingRegressor())
                                        ])

sklearn_estimators_dict = {
    "Ridge Regression": estimator_ridge,
    "Decision Tree": estimator_decision_tree,
    "Random Forest": estimator_random_forest,
    "Gradient Boosting": estimator_gradient_boosting
}
results_dict = {}
for estimator_name, estimator in sklearn_estimators_dict.items():
    print(f"Start training {estimator_name}...")
    # perform cross-validation
    predictions = cross_val_predict(estimator, X_train, y_train, cv=5)

    # get scores training
    r2_score_train = r2_score(y_train, predictions)
    mse_train = mean_squared_error(y_train, predictions)
    
    results_dict[estimator_name] = (r2_score_train, mse_train)
    print(f"MSE = {mse_train} | R-Squared = {r2_score_train}")<jupyter_output>Start training Ridge Regression...
MSE = 0.25024006036909535 | R-Squared = -0.09920087746837414
Start training Decision Tree...
MSE = 0.3289060082017951 | R-Squared = -0.4447477845345136
Start training Random Forest...
MSE = 0.19939648851705943 | R-Squared = 0.1241338624248004
Start training Gradient Boosting...
MSE = 0.19662124075198653 | R-Squared = 0.13632437570257394<jupyter_text>Boost Gradient Booster Apply hyperparameter optimization on gradient booster because of the best performance in comparison to all other approaches.<jupyter_code>from sklearn.model_selection import GridSearchCV

parameters = {
    "learning_rate": [0.001, 0.01, 0.1, 0.5],
    "n_estimators": [50, 100, 300, 500],
    "max_depth": [3, 5, 10],
    "min_samples_split": [2, 3, 4]
 }
# scale data
scaler = StandardScaler()
X_train_s = scaler.fit_transform(X_train)
X_test_s = scaler.transform(X_test)
clf = GridSearchCV(GradientBoostingRegressor(), parameters, n_jobs=-1, cv=5, verbose=2)
clf.fit(X_train_s, y_train)
df_opt_results = pd.DataFrame.from_dict(clf.cv_results_)
df_opt_results.head()
final_estimator = clf.best_estimator_

best_score = clf.best_score_
print(f"Best Score: {best_score}")

best_params = clf.best_params_
print(f"Best Params: {best_params}")<jupyter_output>Best Score: 0.1312062495393764
Best Params: {'learning_rate': 0.1, 'max_depth': 3, 'min_samples_split': 4, 'n_estimators': 100}<jupyter_text>Predict on Testset<jupyter_code>final_preds = np.round(final_estimator.predict(X_test_s), 2)

hold_out_mse = mean_squared_error(y_test, final_preds)
print(f"MSE on hold out test set: {hold_out_mse}")
plt.scatter(y_test, final_preds)

plt.xlabel("Ground Truth")
plt.ylabel("Prediction")
plt.title("Ground Truth vs. Prediction")
plt.show()<jupyter_output><empty_output><jupyter_text>Get Feature Importances<jupyter_code>feat_importances = pd.Series(final_estimator.feature_importances_, index=X_train.columns)
feat_importances.nlargest(10).plot(kind='barh')
plt.title("Feature Importance of ten most important features.", size=17)
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10376508737164475, 'fraction_numerical': 0.04737885065753918, 'mean_word_length': 4.015356820234869, 'pattern_counts': {'":': 8, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '97927', 'license': 'MIT', 'n_tokens_mistral': 2026, 'n_tokens_neox': 1792, 'n_words': 363, 'path': 'Regression/Train_and_Predict.ipynb', 'repo_name': 'patrickbrus/IBM_Machine_Learning_Professional', 'size': '5551'}