<jupyter_start><jupyter_text>Divide dataset into train and validation (test) samples<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)
scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)<jupyter_output><empty_output><jupyter_text>LinearRegression<jupyter_code>lr = LinearRegression()
lr_params = {
    'fit_intercept': [False, True]
}
grid_lr = GridSearchCV(lr, lr_params, cv=10, scoring='r2')
grid_lr.fit(X_train, y_train)
print(grid_lr.best_params_)
print(grid_lr.best_score_)
print(grid_lr.best_estimator_)<jupyter_output>{'fit_intercept': True}
0.7049331148173825
LinearRegression(copy_X=True, fit_intercept=True, n_jobs=None, normalize=False)<jupyter_text>DecisionTreeRegressor<jupyter_code>tree = DecisionTreeRegressor()
tree_params = {
    'max_depth': list(range(1, 11)),
    'min_samples_split': [0.1, 0.2, 0.5, 1.0],
    'criterion': ['mse', 'mae']
}
grid_tree = GridSearchCV(tree, tree_params, cv=10, scoring='r2')
grid_tree.fit(X_train, y_train)
print(grid_tree.best_params_)
print(grid_tree.best_score_)
print(grid_tree.best_estimator_)<jupyter_output>{'criterion': 'mse', 'max_depth': 10, 'min_samples_split': 0.1}
0.7543606899016064
DecisionTreeRegressor(criterion='mse', max_depth=10, max_features=None,
                      max_leaf_nodes=None, min_impurity_decrease=0.0,
                      min_impurity_split=None, min_samples_leaf=1,
                      min_samples_split=0.1, min_weight_fraction_leaf=0.0,
                      presort=False, random_state=None, splitter='best')<jupyter_text>KNeighborsRegressor<jupyter_code>knn = KNeighborsRegressor()
knn_params = {
    'n_neighbors':list(range(1, 20)),
    'weights': ['uniform', 'distance'],
    'algorithm' : ['auto', 'ball_tree', 'kd_tree', 'brute'],
    'leaf_size':list(range(1, 20))
}
grid_knn = GridSearchCV(knn, knn_params, cv=10, scoring='r2')
grid_knn.fit(X_train, y_train)
print(grid_knn.best_params_)
print(grid_knn.best_score_)
print(grid_knn.best_estimator_)<jupyter_output>{'algorithm': 'brute', 'leaf_size': 1, 'n_neighbors': 11, 'weights': 'distance'}
0.7373836764305484
KNeighborsRegressor(algorithm='brute', leaf_size=1, metric='minkowski',
                    metric_params=None, n_jobs=None, n_neighbors=11, p=2,
                    weights='distance')<jupyter_text>RandomForestRegressor<jupyter_code>forest = RandomForestRegressor()
forest_params = {
    'n_estimators': [5, 10, 20, 30, 40],
    'max_depth': [None, 1, 2, 5, 10, 25],
    'min_samples_leaf': list(range(1, 10)),
    'max_features': list(range(1, 10)),
    'criterion': ['mse', 'friedman_mse', 'mae']
}
grid_forest = GridSearchCV(forest, forest_params, cv=10, scoring='r2')
grid_forest.fit(X_train, y_train)
print(grid_forest.best_params_)
print(grid_forest.best_score_)
print(grid_forest.best_estimator_)<jupyter_output>{'criterion': 'mae', 'max_depth': None, 'max_features': 7, 'min_samples_leaf': 1, 'n_estimators': 40}
0.8614062253775506
RandomForestRegressor(bootstrap=True, criterion='mae', max_depth=None,
                      max_features=7, max_leaf_nodes=None,
                      min_impurity_decrease=0.0, min_impurity_split=None,
                      min_samples_leaf=1, min_samples_split=2,
                      min_weight_fraction_leaf=0.0, n_estimators=40,
                      n_jobs=None, oob_score=False, random_state=None,
                      verbose=0, warm_start=False)<jupyter_text>GradientBoostingRegressor<jupyter_code>gboost = GradientBoostingRegressor()
gboost_params = {
    'n_estimators': [5, 10, 20, 30, 40],
    'max_features': list(range(1, 10)),
    'max_depth': [None ,1, 2, 5, 10, 25],
    'learning_rate': [0.1, 0.3, 0.5, 0.7],
    'min_samples_leaf': list(range(1, 10))
}
grid_gboost = GridSearchCV(gboost, gboost_params, cv=10, scoring='r2')
grid_gboost.fit(X_train, y_train)
print(grid_gboost.best_params_)
print(grid_gboost.best_score_)
print(grid_gboost.best_estimator_)<jupyter_output>{'learning_rate': 0.3, 'max_depth': None, 'max_features': 5, 'min_samples_leaf': 3, 'n_estimators': 20}
0.8700005297803008
GradientBoostingRegressor(alpha=0.9, criterion='friedman_mse', init=None,
                          learning_rate=0.3, loss='ls', max_depth=None,
                          max_features=5, max_leaf_nodes=None,
                          min_impurity_decrease=0.0, min_impurity_split=None,
                          min_samples_leaf=3, min_samples_split=2,
                          min_weight_fraction_leaf=0.0, n_estimators=20,
                          n_iter_no_change=None, presort='auto',
                          random_state=None, subsample=1.0, tol=0.0001,
                          validation_fraction=0.1, verbose=0, warm_start=False)<jupyter_text>Compare the results on the validation sample<jupyter_code>estimators = {
    'GradientBoostingRegressor': grid_gboost,
    'RandomForestRegressor': grid_forest,
    'DecisionTreeRegressor': grid_tree,
    'KNeighborsRegressor': grid_knn,
    'LinearRegression': grid_lr
}
for key in estimators:
    value = estimators[key]
    print(
    '{0}:\n    {1}{2}\n    {3}{4}\n\n'.format(
        key, 
        "CV R^2: ", value.best_score_, 
        "Validation R^2: ", value.best_estimator_.score(X_test, y_test)
    )
)
pass<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1353411414761108, 'fraction_numerical': 0.04480386688975646, 'mean_word_length': 4.390781563126253, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '367007', 'license': 'MIT', 'n_tokens_mistral': 2192, 'n_tokens_neox': 1997, 'n_words': 290, 'path': 'bilder/Classic Data Science and Machine Learning/Improved model quality, Advanced algorithms (GridSearchCV, best_params, best_score, best_estimator).ipynb', 'repo_name': 'Alex110117/data_analysis', 'size': '5379'}