<jupyter_start><jupyter_text>1) Use all features<jupyter_code># TODO nrows
# data_all = pd.read_csv('data/training_set_VU_DM_clean.csv', sep=';', nrows=10*1000)
data_all = pd.read_csv('data/training_set_VU_DM_clean.csv', sep=';', nrows=100*1000)
# data_all = pd.read_csv('data/training_set_VU_DM_clean.csv', sep=';')

data_test_unlabelled = pd.read_csv('data/test_set_VU_DM_clean.csv', sep=';', nrows=1000)
for k in data_all.columns:
    if data_all[k].isna().sum() > 0:
#         print('rm %0.4f' % (data_all[k].isna().sum() / data_all.shape[0]), k)
        data_all.drop(columns=[k], inplace=True)

gross_booking_scaler = preprocessing.RobustScaler(copy=False)

# set aside some labelled data for testing (based on srch_id)
ids = data_all.srch_id.unique()
ids_train, ids_test = train_test_split(ids, test_size=0.5, random_state=123)
data = data_all[data_all.srch_id.isin(ids_train)]
data_test = data_all[data_all.srch_id.isin(ids_test)]

k = 'gross_bookings_usd'
data_all.loc[data.index, k] = gross_booking_scaler.fit_transform(data[[k]].values)
data_all.loc[data_test.index, k] = gross_booking_scaler.transform(data_test[[k]].values)
# add noise to reduce dependency on this attr
data_all.loc[data.index, k] += np.random.normal(loc=0, scale=0.01, size=data[k].size)

# split cross validation folds
folds = util.data.cv_folds_for_sklearn(data, n_cv_folds=5, resampling_ratio=0)

# Use the full (pre-splitted) dataset because we use custom indices
# x_train, y_train will depend on the folds
x, y = util.data.split_xy(data_all)

x_test, y_test = util.data.split_xy(data_test)
data.shape, data_test.shape, ids.size # 20086
# TODO
# https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.RandomizedSearchCV.html#sklearn.model_selection.RandomizedSearchCV
    
kwargs = {'random_state': 1234}
models = {}
params = {}
models['GBoost reg'] = xgb.XGBRegressor(criterion='friedman_mse', loss='ls', base_score=0, tol=1e-4, n_jobs=2, **kwargs)
models['GBoost ndcg'] = xgb.XGBRegressor(objective='rank:ndcg', criterion='friedman_mse', 
                                         loss='ls', base_score=0, tol=1e-4, n_jobs=2, **kwargs)
params['GBoost ndcg'] = {'n_estimators': [100, 500, 1000],
                    'learning_rate': [0.1, 0.01, 0.001],
                    'max_depth': [1,2,3,4]
                   }
params['GBoost reg'] = params['GBoost ndcg']
models['Ensemble Random Forest'] = sklearn.ensemble.RandomForestRegressor(n_jobs=2)
params['Ensemble Random Forest'] = {'n_estimators': [1000]}
models['AdaBoost'] = sklearn.ensemble.AdaBoostRegressor()
params['AdaBoost'] = {'learning_rate':[0.2, 0.1, 0.01],
                     'n_estimators': [50, 100, 500],
                      'loss': ['linear', 'square']
                     }


scores = ['neg_mean_squared_error'] # TODO add ndcg here
print('shape', data_all.shape)
for k, model in models.items():
    for score in scores:
        print("\n# %s\t (score: %s)" % (k, score))
        assert k in params.keys(), 'models and params should have the same keys'
        clf = sklearn.model_selection.GridSearchCV(model, params[k], cv=folds, scoring=score, iid=True, n_jobs=2)
        s = 1. # scalar for score
        clf.fit(x, y/s) # cv/folds is used to select folds
        print("\tBest params (train)", clf.best_params_)
        y_true, y_pred = y_test/s, clf.predict(x_test)
        mse =  sklearn.metrics.mean_squared_error(y_true, y_pred)
        print(colored('\tmse: %0.4f' % mse, 'green'))
#         ndcg = util.ndcg.ndcg(x_test, y_test, y_pred)
#         print(colored('\tndcg: %0.4f' % ndcg, 'blue'))
        print('\t', np.median(y_pred), np.max(y_pred))

print('done')<jupyter_output>shape (100000, 85)

# Ensemble Random Forest	 (score: neg_mean_squared_error)
	Best params (train) {'n_estimators': 500}
[32m	mse: 23.5993[0m
	 5.0 5.0
done<jupyter_text>2) Use a subset of features<jupyter_code># TODO nrows
data_all = pd.read_csv('data/training_set_VU_DM_clean.csv', sep=';', nrows=1000*1000)
# data_all = pd.read_csv('data/training_set_VU_DM_clean.csv', sep=';')

data_test_unlabelled = pd.read_csv('data/test_set_VU_DM_clean.csv', sep=';', nrows=1000)
features = [k for k in data_all.columns if 
            'score' in k or 
            'srch_id' in k or 
            'prop_id' in k or 
            'booking_bool' in k or
            'click_bool' in k or
            'unavailable_comp' in k or
            'available_comp' in k or
            'visitor_hist_starrating' in k or
            'delta_starrating' in k or
            'visitor_hist_adr_usd_log' in k or
            'price_usd_log' in k or
            'month' in k or
            'starrating' in k or
            'promotion' in k or
            'srch_length' in k or
            'distance' in k or
            'gross_booking' in k
           ]
features
for k in data_all.columns:
    if data_all[k].isna().sum() > 0:
#         print('rm %0.4f' % (data_all[k].isna().sum() / data_all.shape[0]), k)
        data_all.drop(columns=[k], inplace=True)

gross_booking_scaler = preprocessing.RobustScaler(copy=False)

# set aside some labelled data for testing (based on srch_id)
ids = data_all.srch_id.unique()
ids_train, ids_test = train_test_split(ids, test_size=0.5, random_state=123)
data = data_all[data_all.srch_id.isin(ids_train)]
data_test = data_all[data_all.srch_id.isin(ids_test)]

k = 'gross_bookings_usd'
data_all.loc[data.index, k] = gross_booking_scaler.fit_transform(data[[k]].values)
data_all.loc[data_test.index, k] = gross_booking_scaler.transform(data_test[[k]].values)
# add noise to reduce dependency on this attr
data_all.loc[data.index, k] += np.random.normal(loc=0, scale=0.01, size=data[k].size)

# split cross validation folds
folds = util.data.cv_folds_for_sklearn(data, n_cv_folds=5, resampling_ratio=0)

# Use the full (pre-splitted) dataset because we use custom indices
# x_train, y_train will depend on the folds
x, y = util.data.split_xy(data_all)

x_test, y_test = util.data.split_xy(data_test)
kwargs = {'random_state': 1234}
models = {}
params = {}
models['GBoost reg'] = xgb.XGBRegressor(criterion='friedman_mse', loss='ls', base_score=0, tol=1e-4, n_jobs=2, **kwargs)
models['GBoost ndcg'] = xgb.XGBRegressor(objective='rank:ndcg', criterion='friedman_mse', 
                                         loss='ls', base_score=0, tol=1e-4, n_jobs=2, **kwargs)
params['GBoost reg'] = {'n_estimators': [100, 500, 1000],
                    'learning_rate': [0.2, 0.1, 0.01, 0.001],
                    'max_depth': [1,2,3,4]
                   }
params['GBoost ndcg'] = params['GBoost reg']
# models['Ensemble Random Forest'] = sklearn.ensemble.RandomForestClassifier(n_jobs=2)
# params['Ensemble Random Forest'] = {'n_estimators': [100]}
# models['AdaBoost'] = sklearn.ensemble.AdaBoostRegressor()
# params['AdaBoost'] = {'learning_rate':[0.2, 0.1, 0.01],
#                      'n_estimators': [50, 100, 500],
#                       'loss': ['linear', 'square']
#                      }

scores = ['neg_mean_squared_error'] # TODO add ndcg here
for k, model in models.items():
    for score in scores:
        print("\n# %s\t (score: %s)" % (k, score))
        assert k in params.keys(), 'models and params should have the same keys'
        clf = sklearn.model_selection.GridSearchCV(model, params[k], cv=folds, scoring=score, iid=True, n_jobs=2)
        clf.fit(x, y) # cv/folds is used to select folds
        print("\tBest params (train)", clf.best_params_)
        y_true, y_pred = y_test, clf.predict(x_test)
        mse =  sklearn.metrics.mean_squared_error(y_true, y_pred)
        print(colored('\tmse: %0.4f' % mse, 'green'))
        ndcg = util.ndcg.ndcg(x_test, y_test, y_pred)
        print(colored('\tndcg: %0.4f' % ndcg, 'blue'))

print('done')<jupyter_output># GBoost reg	 (score: neg_mean_squared_error)
	Best params (train) {'learning_rate': 0.1, 'max_depth': 2, 'n_estimators': 100}
[32m	mse: 0.4282[0m
[34m	ndcg: 0.4022[0m

# GBoost ndcg	 (score: neg_mean_squared_error)
	Best params (train) {'learning_rate': 0.2, 'max_depth': 1, 'n_estimators': 500}
[32m	mse: 0.7070[0m
[34m	ndcg: 0.4022[0m
done
{'fraction_non_alphanumeric': 0.14950617283950618, 'fraction_numerical': 0.03160493827160494, 'mean_word_length': 3.6133257403189067, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '518502', 'license': 'MIT', 'n_tokens_mistral': 3262, 'n_tokens_neox': 3052, 'n_words': 625, 'path': 'A2-Compare-xgb-objectives.ipynb', 'repo_name': 'voschezang/Data-Mining', 'size': '8100'}