<jupyter_start><jupyter_text>Stacking<jupyter_code>import pandas as pd, numpy as np, os
from sklearn.metrics import mean_squared_error
import matplotlib.pyplot as plt
PATH = '../preds/'
FILES = os.listdir(PATH)

oof_files = np.sort([f for f in FILES if 'oof' in f and 'extra2' in f and 'five' not in f])
oof_preds = [np.load(PATH + k) for k in oof_files]

print(f'We have {len(oof_preds)} OOF files...')
print(oof_files)<jupyter_output>We have 4 OOF files...
['model_four_extra2_oof.npy' 'model_one_extra2_oof.npy'
 'model_three_extra2_oof.npy' 'model_two_extra2_oof.npy']<jupyter_text>OOF predictions<jupyter_code>x = np.vstack(oof_preds).T
x.shape
x<jupyter_output><empty_output><jupyter_text>Statistical properties of OOF predictions<jupyter_code>preds_min = x.min(axis=1).reshape(-1, 1)
preds_max = x.max(axis=1).reshape(-1, 1)
preds_range = preds_max - preds_min
preds_mean = x.mean(axis=1).reshape(-1, 1)
preds_median = np.median(x, axis=1).reshape(-1, 1)
preds_std = x.std(axis=1).reshape(-1, 1)
from scipy.stats import skew
preds_skew = skew(x, axis=1).reshape(-1, 1)


x = np.concatenate([x, preds_min, preds_max, preds_range, preds_mean, preds_median, preds_std, preds_skew], axis=1)
x.shape<jupyter_output><empty_output><jupyter_text>Under 20<jupyter_code>under_20 = np.load("../preds/model_five_oof.npy").reshape(-1, 1)
under_20 = np.where(under_20 > 0.4, 1, 0)
x = np.concatenate([x, under_20], axis=1)
x.shape
yup = y_true.reshape(-1, 1)[np.where(under_20 > 0.3, True, False)]
(yup < 20).sum() #/ yup.shape[0]
np.where(under_20 > 0.3, True, False)
(y_true[np.where(under_20 > 0.3, 1, 0)] < 20).sum()<jupyter_output><empty_output><jupyter_text>Meta-data provided by organizers<jupyter_code>df = pd.read_csv('../data/train_folds.csv')
y_true = []
other_feats = []
for i in range(5):
    y_true.extend(df[df.kfold == i].Pawpularity.values.tolist())
    other_feats.extend(df[df.kfold == i][[
        "Subject Focus", "Eyes", "Face", "Near", "Action", "Accessory", "Group", "Collage", "Human", "Occlusion", "Info", "Blur"
    ]].values)
y_true = np.array(y_true)
x_feats = np.array(other_feats)
x_feats.shape
x_feats
x = np.concatenate([x, x_feats], axis=1)
x.shape<jupyter_output><empty_output><jupyter_text>More image meta-data<jupyter_code># from PIL import Image


# img_feats = [] 
# for i in range(5):
#     image_fpaths = [f'../data/train/{i}.jpg' for i in df[df.kfold == i].Id.tolist()]
#     img_shape = np.array([np.array(Image.open(img).size) for img in image_fpaths])
#     img_aspect_ratio = (img_shape[:, 0] / img_shape[:, 1]).reshape(-1, 1)
#     img_mean = np.array([np.array(Image.open(img)).mean() for img in image_fpaths]).reshape(-1, 1)
#     img_feats.extend(np.hstack([img_shape, img_aspect_ratio, img_mean]))

# img_feats = np.array(img_feats)
# img_feats
# x = np.concatenate([x, img_feats], axis=1)
# x.shape<jupyter_output><empty_output><jupyter_text>Set up Cross Validation<jupyter_code>from sklearn.model_selection import PredefinedSplit

cv = PredefinedSplit(df.kfold)<jupyter_output><empty_output><jupyter_text>Train meta-model  GradientBoostingRegressor<jupyter_code>from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import GradientBoostingRegressor

reg = GradientBoostingRegressor()
parameters = {
    'n_estimators': [100, 300, 500],
    'learning_rate':[0.01, 0.01, 0.1, 1],
    'max_depth':[3, 5, 7],
    'max_features': ["auto", "sqrt", "log2"]
}
clf = GridSearchCV(reg, parameters, n_jobs=-1, cv=cv, scoring='neg_root_mean_squared_error')
grid = clf.fit(x, y_true)
grid.best_params_
from sklearn.linear_model import Lasso
from sklearn.svm import SVR
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import AdaBoostRegressor
 
y_pred = []
y_real = []

for train_idx, val_idx in cv.split():
    x_train, y_train = x[train_idx, :], y_true[train_idx]
    
    lr = GradientBoostingRegressor(**grid.best_params_)
    lr.fit(X=x_train, y=y_train)
    
    x_val = x[val_idx, :]
    preds = lr.predict(x_val)
    
    y_pred.extend(preds)
    y_real.extend(y_true[val_idx])
np.min(y_pred), np.max(y_pred)
mean_squared_error(y_real, np.clip(y_pred, a_min=0, a_max=100), squared=False)
plt.hist(y_pred, bins=50, label='predictions');
plt.hist(y_real, bins=50, alpha=.25, label='pawpularity');
plt.legend();<jupyter_output><empty_output><jupyter_text>RandomForest<jupyter_code>reg = RandomForestRegressor()
parameters = {
    'n_estimators': [100, 300, 500],
    'max_depth':[3, 5, 7],
    'max_features': ["auto", "sqrt", "log2"]
}
clf = GridSearchCV(reg, parameters, n_jobs=-1, cv=cv, scoring='neg_root_mean_squared_error')
grid = clf.fit(x, y_true)
grid.best_params_
from sklearn.linear_model import Lasso
from sklearn.svm import SVR
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.ensemble import AdaBoostRegressor
 
y_pred = []
y_real = []

for train_idx, val_idx in cv.split():
    x_train, y_train = x[train_idx, :], y_true[train_idx]
    
    lr = RandomForestRegressor(**grid.best_params_)
    lr.fit(X=x_train, y=y_train)
    
    x_val = x[val_idx, :]
    preds = lr.predict(x_val)
    
    y_pred.extend(preds)
    y_real.extend(y_true[val_idx])
np.min(y_pred), np.max(y_pred)
mean_squared_error(y_real, np.clip(y_pred, a_min=0, a_max=100), squared=False)
plt.hist(y_pred, bins=50, label='predictions');
plt.hist(y_real, bins=50, alpha=.25, label='pawpularity');
plt.legend();<jupyter_output><empty_output><jupyter_text>Lasso<jupyter_code>from sklearn.linear_model import LassoCV
lr = LassoCV(fit_intercept=True, normalize=True, cv=cv, random_state=1010)
lr.fit(X=x, y=y_true)
lr.alpha_, lr.coef_
y_pred = np.clip(lr.predict(X=x), a_min=0, a_max=100)
plt.hist(y_pred, bins=50, label='predictions');
plt.hist(y_true, bins=50, alpha=.25, label='pawpularity');
plt.legend();
mean_squared_error(y_true, y_pred, squared=False)
all = []
for k in range(x.shape[1]):
    rmse = mean_squared_error(y_true, x[:, k], squared=False)
    all.append(rmse)
    print('Model %i has OOF RMSE = %.4f'%(k, rmse))
    
m = [np.argmin(all)]; w = []<jupyter_output>Model 0 has OOF RMSE = 17.3479
Model 1 has OOF RMSE = 17.2234
Model 2 has OOF RMSE = 17.2713
Model 3 has OOF RMSE = 17.2824
Model 4 has OOF RMSE = 17.3218
Model 5 has OOF RMSE = 17.3801
Model 6 has OOF RMSE = 39.6572
Model 7 has OOF RMSE = 17.1871
Model 8 has OOF RMSE = 17.1922
Model 9 has OOF RMSE = 41.8478
Model 10 has OOF RMSE = 43.2772
Model 11 has OOF RMSE = 43.1726
Model 12 has OOF RMSE = 43.2540
Model 13 has OOF RMSE = 42.6031
Model 14 has OOF RMSE = 42.4844
Model 15 has OOF RMSE = 42.5227
Model 16 has OOF RMSE = 43.2686
Model 17 has OOF RMSE = 43.2167
Model 18 has OOF RMSE = 43.1621
Model 19 has OOF RMSE = 43.2344
Model 20 has OOF RMSE = 43.1322
Model 21 has OOF RMSE = 43.1276
Model 22 has OOF RMSE = 43.2248
Model 23 has OOF RMSE = 43.2187
{'fraction_non_alphanumeric': 0.14695601514710166, 'fraction_numerical': 0.04777162831342849, 'mean_word_length': 5.220108695652174, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '654755', 'license': 'MIT', 'n_tokens_mistral': 3160, 'n_tokens_neox': 2833, 'n_words': 591, 'path': 'nbs/ensembling.ipynb', 'repo_name': 'IamGianluca/petfinder-pawpularity-score', 'size': '6866'}