<jupyter_start><jupyter_text>Hyperparameter optimization<jupyter_code>from bayes_opt import BayesianOptimization
def train_rf(max_depth,n_estimators,max_features,colsample_bytree,learning_rate):
    params = {
        'max_depth': int(max_depth),
        'n_estimators': int(n_estimators),
        'max_features': max_features
    }
    clf = RandomForestClassifier(**params, random_state=42)
    clf.fit(train2[cnt_cols+cat_num_cols].iloc[train_index2], train2.RESULT.iloc[train_index2])
    pred = clf.predict(train2[cnt_cols+cat_num_cols].iloc[test_index2])
    score = f1_score(train2.RESULT.iloc[test_index2],pred)
    print("F1 {:.3f} params {}".format(score, params))
    return score
def train_gbm(max_depth,n_estimators,max_features,colsample_bytree,learning_rate):
    params = {
        'max_depth': int(max_depth),
        'n_estimators': int(n_estimators),
        'max_features': max_features,
        'learning_rate': learning_rate
    }
    clf = GradientBoostingClassifier(**params, random_state=42)
    clf.fit(train2[cnt_cols+cat_num_cols].iloc[train_index2], train2.RESULT.iloc[train_index2])
    pred = clf.predict(train2[cnt_cols+cat_num_cols].iloc[test_index2])
    score = f1_score(train2.RESULT.iloc[test_index2],pred)
    print("F1 {:.3f} params {}".format(score, params))
    return score
def train_xgb(max_depth,n_estimators,max_features,colsample_bytree,learning_rate):
    params = {
        'max_depth': int(max_depth),
        'n_estimators': int(n_estimators),
        'colsample_bytree': colsample_bytree,
        'learning_rate': learning_rate
    }
    clf = XGBClassifier(**params, random_state=42)
    clf.fit(train2[cnt_cols+cat_num_cols].iloc[train_index2], train2.RESULT.iloc[train_index2])
    pred = clf.predict(train2[cnt_cols+cat_num_cols].iloc[test_index2])
    score = f1_score(train2.RESULT.iloc[test_index2],pred)
    print("F1 {:.3f} params {}".format(score, params))
    return score
def train_lgb(max_depth,n_estimators,max_features,colsample_bytree,learning_rate):
    params = {
        'max_depth': int(max_depth),
        'n_estimators': int(n_estimators),
        'colsample_bytree': colsample_bytree,
        'learning_rate': learning_rate
    }
    clf = LGBMClassifier(**params, random_state=42)
    clf.fit(train2[cnt_cols+cat_num_cols].iloc[train_index2], train2.RESULT.iloc[train_index2]
    pred = clf.predict(train2[cnt_cols+cat_num_cols].iloc[test_index2])
    score = f1_score(train2.RESULT.iloc[test_index2],pred)
    print("F1 {:.3f} params {}".format(score, params))
    return score
bounds = {
    'max_depth':(5,20),
    'n_estimators': (100,5000),
    'max_features':(.5,1),
    'colsample_bytree':(.5,1),
    'learning_rate':(.05,.15)
}
optimizer = BayesianOptimization(
    f=train_rf,
    pbounds=bounds,
    random_state=1,
)
optimizer.maximize(init_points=10, n_iter=20)
optimizer = BayesianOptimization(
    f=train_lgb,
    pbounds=bounds,
    random_state=1,
)
optimizer.maximize(init_points=10, n_iter=20)
optimizer = BayesianOptimization(
    f=train_xgb,
    pbounds=bounds,
    random_state=1,
)
optimizer.maximize(init_points=10, n_iter=20)
optimizer = BayesianOptimization(
    f=train_gbm,
    pbounds=bounds,
    random_state=1,
)
optimizer.maximize(init_points=10, n_iter=20)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13171471927162368, 'fraction_numerical': 0.027314112291350532, 'mean_word_length': 5.047706422018349, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '458490', 'license': 'MIT', 'n_tokens_mistral': 1284, 'n_tokens_neox': 1240, 'n_words': 146, 'path': 'Mortgage Propensity Modelling/notebooks/cat_feature_hyperoptimization.ipynb', 'repo_name': 'victor7246/Competition-Solutions', 'size': '3295'}