<jupyter_start><jupyter_text>Tuning XGBoost Parameters<jupyter_code>class BlockingTimeSeriesSplit():
    def __init__(self, n_splits):
        self.n_splits = n_splits
    
    def get_n_splits(self, X, y, groups):
        return self.n_splits
    
    def split(self, X, y=None, groups=None):
        n_samples = len(X)
        k_fold_size = n_samples // self.n_splits
        indices = np.arange(n_samples)

        margin = 0
        for i in range(self.n_splits):
            start = i * k_fold_size
            stop = start + k_fold_size
            mid = int(0.8 * (stop - start)) + start
            yield indices[start: mid], indices[mid + margin: stop]
pm25_supervised_df = timeseries_to_supervised(donggaocun_aq.set_index('utc_datetime'), 16, dropnan=False)
pm10_supervised_df = timeseries_to_supervised(donggaocun_aq.set_index('utc_datetime'), 15, dropnan=False)
O3_supervised_df = timeseries_to_supervised(donggaocun_aq.set_index('utc_datetime'), 10, dropnan=False)

X_train_pm25, y_train_pm25, X_valid_pm25, y_valid_pm25 = generate_train_test_data(pm25_supervised_df, split_ratio=0.8, target_column='PM2.5')
X_train_pm10, y_train_pm10, X_valid_pm10, y_valid_pm10 = generate_train_test_data(pm10_supervised_df, split_ratio=0.8, target_column='PM10')
X_train_O3, y_train_O3, X_valid_O3, y_valid_O3 = generate_train_test_data(O3_supervised_df, split_ratio=0.8, target_column='O3')
X_train_pm25.head()
# n_estimators = [100, 200, 300, 400, 500]
# learning_rate = [0.0001, 0.001, 0.01, 0.1]
# max_depth = [2, 4, 6, 8]

# model = XGBRegressor()
# param_grid = dict(n_estimators=n_estimators, max_depth=max_depth, learning_rate=learning_rate)
# btscv = BlockingTimeSeriesSplit(n_splits=5)
# grid_search = GridSearchCV(model, param_grid, scoring='neg_mean_squared_error', n_jobs=-1, cv=btscv, verbose=1)
# grid_result = grid_search.fit(X_train_pm25, y_train_pm25)
# grid_result.best_params_<jupyter_output><empty_output><jupyter_text>Test XGBoost parameters<jupyter_code># {'learning_rate': 0.01, 'max_depth': 2, 'n_estimators': 500}
pm25_xg_model = XGBRegressor(n_estimators=500, learning_rate=0.01, max_depth=2)
pm25_xg_model.fit(X_train_pm25, y_train_pm25)

pm25_non_na_values = y_valid_pm25 != -1
y_valid_pm25 = y_valid_pm25[pm25_non_na_values]
X_valid_pm25 = X_valid_pm25[pm25_non_na_values]

print_score(pm25_xg_model, X_train_pm25, y_train_pm25, X_valid_pm25, y_valid_pm25)
fig, ax = plt.subplots(figsize=(5, 10))
plot_importance(pm25_xg_model, height=0.9, max_num_features=10, ax=ax)
# Save xgboost pm2.5 results

pm25_predictions = pm25_xg_model.predict(X_valid_pm25)
np.save('pm25_xg_predictions', pm25_predictions)
y_valid_pm25.to_csv('pm25_actual.csv')
pm10_xg_model = XGBRegressor(n_estimators=500, learning_rate=0.01, max_depth=2)
pm10_xg_model.fit(X_train_pm10, y_train_pm10)

pm10_non_na_values = y_valid_pm10 != -1
y_valid_pm10 = y_valid_pm10[pm10_non_na_values]
X_valid_pm10 = X_valid_pm10[pm10_non_na_values]


print_score(pm10_xg_model, X_train_pm10, y_train_pm10, X_valid_pm10, y_valid_pm10)
fig, ax = plt.subplots(figsize=(5, 10))
plot_importance(pm10_xg_model, height=0.9, max_num_features=10, ax=ax)
pm10_xg_predictions = pm10_xg_model.predict(X_valid_pm10)
np.save('pm10_xg_predictions', pm10_xg_predictions)
y_valid_pm10.to_csv('pm10_actual.csv')
O3_xg_model = XGBRegressor(n_estimators=500, learning_rate=0.01, max_depth=2)
O3_xg_model.fit(X_train_O3, y_train_O3)

O3_non_na_values = y_valid_O3 != -1
y_valid_O3 = y_valid_O3[O3_non_na_values]
X_valid_O3 = X_valid_O3[O3_non_na_values]

print_score(O3_xg_model, X_train_O3, y_train_O3, X_valid_O3, y_valid_O3)
fig, ax = plt.subplots(figsize=(5, 10))
plot_importance(O3_xg_model, height=0.9, max_num_features=10, ax=ax)
O3_xg_predictions = O3_xg_model.predict(X_valid_O3)
np.save('O3_xg_predictions', O3_xg_predictions)
y_valid_O3.to_csv('O3_actual.csv')<jupyter_output><empty_output><jupyter_text>Tuning lightgbm Parameters<jupyter_code># lgb_grid_params = {
#     'learning_rate': [0.001, 0.01, 0.04, 0.07, 0.1],
#     'n_estimators': [50, 100, 150, 200, 300],
#     'num_leaves': [6, 8,12,16],
#     'boosting_type' : ['gbdt'],
#     'min_data_in_leaf' : [30, 50, 80, 100],
#     }

# lgb_model = lgb.LGBMRegressor()
# lgb_btscv = BlockingTimeSeriesSplit(n_splits=5)
# grid_search_lgb = GridSearchCV(lgb_model, lgb_grid_params, scoring='neg_mean_squared_error', n_jobs=-1, cv=lgb_btscv, verbose=1)
# grid_result = grid_search_lgb.fit(X_train_pm25, y_train_pm25)
# grid_result.best_params_
# lightgbm best params
# {'boosting_type': 'gbdt',
#  'learning_rate': 0.1,
#  'min_data_in_leaf': 30,
#  'n_estimators': 300,
#  'num_leaves': 6}


pm25_lgb_model = lgb.LGBMRegressor(boosting_type='gbdt', learning_rate=0.1, min_data_in_leaf=30, n_estimators=300, num_leaves=6)
pm25_lgb_model.fit(X_train_pm25, y_train_pm25)

print_score(pm25_lgb_model, X_train_pm25, y_train_pm25, X_valid_pm25, y_valid_pm25)
fig, ax = plt.subplots(figsize=(5, 10))
lgb.plot_importance(pm25_lgb_model, height=0.9, max_num_features=10, ax=ax)
pm25_lgb_predictions = pm25_lgb_model.predict(X_valid_pm25)
np.save('pm25_lgb_predictions', pm25_lgb_predictions)
pm10_lgb_model = lgb.LGBMRegressor(boosting_type='gbdt', learning_rate=0.1, min_data_in_leaf=30, n_estimators=300, num_leaves=6)
pm10_lgb_model.fit(X_train_pm10, y_train_pm10)

print_score(pm10_lgb_model, X_train_pm10, y_train_pm10, X_valid_pm10, y_valid_pm10)
fig, ax = plt.subplots(figsize=(5, 10))
lgb.plot_importance(pm10_lgb_model, height=0.9, max_num_features=10, ax=ax)
pm10_lgb_predictions = pm10_lgb_model.predict(X_valid_pm10)
np.save('pm10_lgb_predictions', pm10_lgb_predictions)
O3_lgb_model = lgb.LGBMRegressor(boosting_type='gbdt', learning_rate=0.1, min_data_in_leaf=30, n_estimators=300, num_leaves=6)
O3_lgb_model.fit(X_train_O3, y_train_O3)

print_score(O3_lgb_model, X_train_O3, y_train_O3, X_valid_O3, y_valid_O3)
fig, ax = plt.subplots(figsize=(5, 10))
lgb.plot_importance(O3_lgb_model, height=0.9, max_num_features=10, ax=ax)
O3_lgb_predictions = O3_lgb_model.predict(X_valid_O3)
np.save('O3_lgb_predictions', O3_lgb_predictions)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11931724930247825, 'fraction_numerical': 0.07385524372230429, 'mean_word_length': 7.631728045325779, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '175551', 'license': 'MIT', 'n_tokens_mistral': 2964, 'n_tokens_neox': 2715, 'n_words': 353, 'path': 'notebooks/Gradient_Boosted_Trees.ipynb', 'repo_name': 'mjjimenez/kdd_air_quality_forecasting', 'size': '6093'}