<jupyter_start><jupyter_text>Using XGBRegressor and important features<jupyter_code>from sklearn.grid_search import GridSearchCV
from xgboost import XGBRegressor
params_reg = dict(params)
params_reg.pop("eta")
params_reg.pop('eval_metric')
params_reg.pop('lambda')
reg = XGBRegressor(**params_reg)
reg.fit(train.drop("loss", 1), train.loss)
train_predprob = reg.score()
reg_booster = reg.booster()
figsize(18, 5)
feat_imp = pd.Series(reg.booster().get_fscore()).sort_values(ascending=False)
feat_imp.plot(kind='bar', title='Feature Importances')
important_features = list(feat_imp[feat_imp > 4].index)
print("important features:", important_features)
dtrain_imp = xgb.DMatrix(train[important_features], train.loss)
cvresult = xgb.cv(params, dtrain_imp, nfold=4, num_boost_round=50)
print(cvresult)
params2 = {'base_score': 0.1, 'colsample_bytree': 0.9,
 'eta': 0.3,
 'eval_metric': 'mae',
 'max_depth': 7,
 'min_child_weight': 3,
 'n_estimators': 10,
 'objective': 'reg:linear',
 'seed': 1,
 'silent': True}
regb = xgb.train(params2, dtrain_imp, num_boost_round=50, evals=[(dtrain_imp, "train")])
param_test1 = {
 'max_depth':range(3,10,2),
 'min_child_weight':range(1,6,2)
}
gsearch1 = GridSearchCV(estimator=reg, 
 param_grid = param_test1, scoring='neg_mean_squared_error',n_jobs=4, iid=False, cv=5)
gsearch1.fit(train.drop("loss", 1), train.loss)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16159420289855073, 'fraction_numerical': 0.02681159420289855, 'mean_word_length': 8.794326241134751, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '130581', 'license': 'Apache-2.0', 'n_tokens_mistral': 555, 'n_tokens_neox': 529, 'n_words': 88, 'path': 'xgb_plain_template.ipynb', 'repo_name': 'aliciawyy/notebooks', 'size': '1380'}