<jupyter_start><jupyter_text>Project name: [Predict Future Sales](https://www.kaggle.com/c/competitive-data-science-predict-future-sales)  Version<jupyter_code>__ver__ = "0.6"<jupyter_output><empty_output><jupyter_text>Setup<jupyter_code>import itertools
import warnings
import sys
import pickle

warnings.filterwarnings('ignore')

from sklearn.linear_model import Ridge
from sklearn.metrics import mean_squared_error
import numpy as np
import pandas as pd
import catboost
import xgboost
import lightgbm
import sklearn
import keras
from tensorflow import set_random_seed
from sklearn import preprocessing
from sklearn import ensemble
import matplotlib.pyplot as plt
import statsmodels as sm
%matplotlib inline<jupyter_output><empty_output><jupyter_text>Data load<jupyter_code>test_path = "./raw_data/test.csv.gz"
processed_path = f"./processed_data/data_{__ver__}.pickle"
# load and save ID in index
test = pd.read_csv(test_path).set_index('ID')
data = pd.read_pickle(processed_path)<jupyter_output><empty_output><jupyter_text>Validation strategies for Ensembling * Train - all months except last two in the training set with optimal parameters* Make meta features for last two months* Train metamodel on the first one* Validate metamodel on the second one* Test - a test set<jupyter_code>data.info()
val_range = 2
test_date_block = data.date_block_num.max()

x_train = data[data.date_block_num < test_date_block - val_range].drop("y", axis=1)
y_train = data.loc[data.date_block_num < test_date_block - val_range, "y"]

mask = (test_date_block - val_range <= data.date_block_num) & (data.date_block_num < test_date_block)
x_valid = data[mask].drop("y", axis=1)
y_valid = data.loc[mask, "y"]

x_test = data[data.date_block_num == test_date_block].drop("y", axis=1)
cat_columns = [i for i, col in enumerate(x_train) if not issubclass(x_train[col].dtype.type, np.floating)]
cat_columns<jupyter_output><empty_output><jupyter_text>Make metafeatures  Catboost Set iterations to 115<jupyter_code>clf_params = dict(
    random_state=284704,
    od_type="Iter",
    od_wait=20,
    verbose=10,
    learning_rate=0.1,
    iterations=115,
)

clf = catboost.CatBoostRegressor(**clf_params)

fit_params = dict(
    X=x_train, 
    y=y_train,
    cat_features=cat_columns,
    plot=True
)

clf.fit(**fit_params)
# save predictions for ensemble
valid_pred = pd.DataFrame(clf.predict(x_valid), columns=["catboost"]).clip(0, 20)
test_pred = pd.DataFrame(clf.predict(x_test), index=test.index, columns=["item_cnt_month"]).clip(0, 20)

valid_pred.to_pickle("./processed_data/catboost_valid.pickle")
test_pred.to_pickle("./processed_data/catboost_test.pickle")<jupyter_output><empty_output><jupyter_text>XGBoost Changed default parameters and 58 n_estimators<jupyter_code>clf_params = dict(
    random_state=284704,
    learning_rate=0.1,
    n_estimators=58,
    max_depth=6,
    min_child_weight=396.3160486093314, 
)
clf = xgboost.XGBRegressor(**clf_params)

clf.fit(
    x_train, 
    y_train, 
    eval_metric="rmse", 
    eval_set=[(x_train, y_train)], 
    verbose=10)
# save predictions for ensemble
valid_pred = pd.DataFrame(clf.predict(x_valid), columns=["xgboost"]).clip(0, 20)
test_pred = pd.DataFrame(clf.predict(x_test), index=test.index, columns=["item_cnt_month"]).clip(0, 20)

valid_pred.to_pickle("./processed_data/xgboost_valid.pickle")
test_pred.to_pickle("./processed_data/xgboost_test.pickle")<jupyter_output><empty_output><jupyter_text>LightGBM Set n_estimators 79<jupyter_code>clf_params = dict(
    random_state=284704,
    learning_rate=0.1,
    n_estimators=79,
)
clf = lightgbm.LGBMRegressor(**clf_params)

clf.fit(
    x_train, 
    y_train, 
    early_stopping_rounds=20,
    eval_set=[(x_train, y_train)], 
    verbose=10,
)
# save predictions for ensemble
valid_pred = pd.DataFrame(clf.predict(x_valid), columns=["lightgbm"]).clip(0, 20)
test_pred = pd.DataFrame(clf.predict(x_test), index=test.index, columns=["item_cnt_month"]).clip(0, 20)

valid_pred.to_pickle("./processed_data/lightgbm_valid.pickle")
test_pred.to_pickle("./processed_data/lightgbm_test.pickle")<jupyter_output><empty_output><jupyter_text>RandomForest RandomForest "as is"<jupyter_code>clf_params = dict(
    random_state=284704,
    n_jobs=-1,
    verbose=False,
    n_estimators=100,
)
clf = ensemble.RandomForestRegressor(**clf_params)

clf.fit(
    x_train, 
    y_train
)
# validation RMSE
valid_pred = pd.DataFrame(clf.predict(x_valid), columns=["rf"])
test_pred = pd.DataFrame(clf.predict(x_test), index=test.index, columns=["item_cnt_month"])

sklearn.metrics.mean_squared_error(valid_pred, y_valid) ** 0.5
# clip and save predictions for ensemble
valid_pred.clip(0, 20).to_pickle("./processed_data/rf_valid.pickle")
test_pred.clip(0, 20).to_pickle("./processed_data/rf_test.pickle")<jupyter_output><empty_output><jupyter_text>LightGBM RF Changed default parameters and 62 n_estimators<jupyter_code>clf_params = dict(
    boosting_type="rf", 
    bagging_freq=1, 
    bagging_fraction=0.06165908963231879, 
    feature_fraction=0.602742610960687,
    random_state=284704,
    n_estimators=62,
)
clf = lightgbm.LGBMRegressor(**clf_params)

clf.fit(
    x_train, 
    y_train, 
    eval_set=[(x_train, y_train)], 
    verbose=10,
)
# save predictions for ensemble
valid_pred = pd.DataFrame(clf.predict(x_valid), columns=["lightgbm_rf"]).clip(0, 20)
test_pred = pd.DataFrame(clf.predict(x_test), index=test.index, columns=["item_cnt_month"]).clip(0, 20)

valid_pred.to_pickle("./processed_data/lightgbm_rf_valid.pickle")
test_pred.to_pickle("./processed_data/lightgbm_rf_test.pickle")<jupyter_output><empty_output><jupyter_text>Simple NN Set epochs 3<jupyter_code># make some data transfomation for NN
nn_data = data.copy()
nn_data.reset_index(drop=True, inplace=True)
nn_data["item_id"] = keras.utils.normalize(data[["item_id"]].values, axis=0)
nn_data = pd.concat([nn_data, pd.DataFrame(keras.utils.to_categorical(data["shop_id"]), dtype='bool')], axis=1)
nn_data.drop("shop_id", axis=1, inplace=True)
# split train, validation, test
train_mask = (data.date_block_num < test_date_block - val_range).values
valid_mask = ((test_date_block - val_range <= data.date_block_num) & (data.date_block_num < test_date_block)).values
test_mast = (data.date_block_num == test_date_block).values

nn_x_train = nn_data[train_mask].drop("y", axis=1)
nn_y_train = nn_data.loc[train_mask, "y"]

nn_x_valid = nn_data[valid_mask].drop("y", axis=1)
nn_y_valid = nn_data.loc[valid_mask, "y"]

nn_x_test = nn_data[test_mast].drop("y", axis=1)
clf = keras.models.Sequential()
nn_num = nn_x_train.shape[1]
clf.add(keras.layers.Dense(int(nn_num / 2), input_dim=nn_num, activation='relu'))
clf.add(keras.layers.Dropout(0.2))
clf.add(keras.layers.Dense(int(nn_num / 4), activation='relu'))
clf.add(keras.layers.Dropout(0.2))
clf.add(keras.layers.Dense(int(nn_num / 8), activation='relu'))
clf.add(keras.layers.Dropout(0.2))
clf.add(keras.layers.Dense(1, activation='relu'))
clf.compile(loss='mse', optimizer='RMSprop', metrics=['mse'])
clf.fit(
    nn_x_train, nn_y_train, 
    epochs=3, batch_size=512)
# save predictions for ensemble
valid_pred = pd.DataFrame(clf.predict(nn_x_valid), columns=["NN"]).clip(0, 20)
test_pred = pd.DataFrame(clf.predict(nn_x_test), index=test.index, columns=["item_cnt_month"]).clip(0, 20)

valid_pred.to_pickle("./processed_data/nn_valid.pickle")
test_pred.to_pickle("./processed_data/nn_test.pickle")<jupyter_output><empty_output><jupyter_text>Save validation target<jupyter_code>nn_y_valid.to_pickle("./processed_data/valid_target.pickle")<jupyter_output><empty_output><jupyter_text>Data load<jupyter_code>test_path = "./raw_data/test.csv.gz"
test = pd.read_csv(test_path).set_index('ID')
catboost_test = pd.read_pickle("./processed_data/catboost_test.pickle")
catboost = pd.read_pickle("./processed_data/catboost_valid.pickle")

lightgbm_rf_test = pd.read_pickle("./processed_data/lightgbm_rf_test.pickle")
lightgbm_rf = pd.read_pickle("./processed_data/lightgbm_rf_valid.pickle")

lightgbm_test = pd.read_pickle("./processed_data/lightgbm_test.pickle")
lightgbm = pd.read_pickle("./processed_data/lightgbm_valid.pickle")

nn_test = pd.read_pickle("./processed_data/nn_test.pickle")
nn = pd.read_pickle("./processed_data/nn_valid.pickle")

rf_test = pd.read_pickle("./processed_data/rf_test.pickle")
rf = pd.read_pickle("./processed_data/rf_valid.pickle")

xgboost_test = pd.read_pickle("./processed_data/xgboost_test.pickle")
xgboost = pd.read_pickle("./processed_data/xgboost_valid.pickle")
valid_y = pd.read_pickle("./processed_data/valid_target.pickle").reset_index(drop=True)<jupyter_output><empty_output><jupyter_text>Split into meta train and valid<jupyter_code>train_mask = catboost.index[:int((len(catboost.index) + 1) / 2)]
valid_mask = catboost.index[int((len(catboost.index) + 1) / 2):]

catboost_meta_train = catboost.loc[train_mask]
catboost_meta_valid = catboost.loc[valid_mask]

lightgbm_rf_meta_train = lightgbm_rf.loc[train_mask]
lightgbm_rf_meta_valid = lightgbm_rf.loc[valid_mask]

lightgbm_meta_train = lightgbm.loc[train_mask]
lightgbm_meta_valid = lightgbm.loc[valid_mask]

nn_meta_train = nn.loc[train_mask]
nn_meta_valid = nn.loc[valid_mask]

rf_meta_train = rf.loc[train_mask]
rf_meta_valid = rf.loc[valid_mask]

xgboost_meta_train = xgboost.loc[train_mask]
xgboost_meta_valid = xgboost.loc[valid_mask]

meta_y_train = valid_y.loc[train_mask]
meta_y_valid = valid_y.loc[valid_mask]<jupyter_output><empty_output><jupyter_text>Stack data and show correlation<jupyter_code>stack_train = pd.concat([
    catboost_meta_train, 
    lightgbm_rf_meta_train, 
    lightgbm_meta_train, 
    nn_meta_train, 
    rf_meta_train, 
    xgboost_meta_train], axis=1).reset_index(drop=True)

stack_valid = pd.concat([
    catboost_meta_valid, 
    lightgbm_rf_meta_valid, 
    lightgbm_meta_valid, 
    nn_meta_valid, 
    rf_meta_valid, 
    xgboost_meta_valid], axis=1).reset_index(drop=True)

stack_test = pd.concat([
    catboost_test, 
    lightgbm_rf_test, 
    lightgbm_test, 
    nn_test, 
    rf_test, 
    xgboost_test], axis=1).reset_index(drop=True)
sns.heatmap(stack_train.corr(), cmap="RdYlGn", annot=True)
# save for hyper optimization
stack_train.to_pickle("./processed_data/meta_train.pickle")
meta_y_train.to_pickle("./processed_data/meta_y_train.pickle")
stack_valid.to_pickle("./processed_data/meta_valid.pickle")
meta_y_valid.to_pickle("./processed_data/meta_y_valid.pickle")<jupyter_output><empty_output><jupyter_text>Ensembling I run hyper_ensembling.py script to select optimal Ridge parameters<jupyter_code>clf_params = {'alpha': 60732.99121614573, 'random_state': 284704}
clf = Ridge(**clf_params)

clf.fit(
    stack_train, 
    meta_y_train,
)
# validation RMSE
mean_squared_error(clf.predict(stack_valid), meta_y_valid) ** 0.5
clf.intercept_
pd.Series(clf.coef_, index=stack_train.columns).sort_values(ascending=False)
# save meta model
with open("./processed_data/meta_model.pickle", "wb") as fout:
    pickle.dump(clf, fout)<jupyter_output><empty_output><jupyter_text>Results<jupyter_code># Catboost     - 0.93440
# LightGBM     - 0.93864
# XGBoost      - 0.94213
# LightGBM RF  - 0.94227
# RandomForest - 0.95773
# Simple NN    - 0.95999

# Ensemble     - 0.93377 (Catboost-0.00063)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13181050573282374, 'fraction_numerical': 0.02373122389120967, 'mean_word_length': 6.536503683858004, 'pattern_counts': {'":': 0, '<': 72, '<?xml version=': 0, '>': 65, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '414610', 'license': 'Unlicense', 'n_tokens_mistral': 4637, 'n_tokens_neox': 4368, 'n_words': 599, 'path': 'solution/Step3 - Ensembling.ipynb', 'repo_name': 'WLM1ke/How_to_Win_Kagglers', 'size': '11251'}