<jupyter_start><jupyter_text>Notes on hyperparamters:  num_leaves, max_depth (between 3 and 12), min_data_in_leaf in hundreds or thousands for big datasets (specifies the minimum number of observations that fit the decision criteria in a leaf.)  n_estimators, learning_rate, max_bin (maybe), lambda_l1, lambda_l2, min_gain_to_split, bagging_fraction, feature_fraction, bagging_freq  Hyperparameter Tuning<jupyter_code>import optuna

def objective(trial, features=features, X=book_df, y=book_df[target], cats=[]):
    param_grid = {"n_estimators": trial.suggest_categorical("n_estimators", [813]),
        "boosting": trial.suggest_categorical("boosting",['dart']),
        "learning_rate": trial.suggest_float("learning_rate", 0.1, 0.2),
        "num_leaves": trial.suggest_int("num_leaves", 511, 750),
        #"max_depth": trial.suggest_int("max_depth", 3, 12),
        "min_data_in_leaf": trial.suggest_int("min_data_in_leaf", 617, 800),
        "max_bin": trial.suggest_int("max_bin", 200, 300),
        #"lambda_l1": trial.suggest_int("lambda_l1", 0, 100, step=5),
        #"lambda_l2": trial.suggest_int("lambda_l2", 0, 100, step=5),
        #"min_gain_to_split": trial.suggest_float("min_gain_to_split", 0, 15),
        "bagging_fraction": trial.suggest_float("bagging_fraction", 0.6, 0.75),
        "bagging_freq": trial.suggest_categorical("bagging_freq", [1,7]),
        "feature_fraction": trial.suggest_float("feature_fraction", 0.8, 0.95)
    }
    
    cv = model_selection.KFold(n_splits=2, shuffle=True, random_state=46) #n_splits is reduced to 2 for time saving
    #kf = cv.split(X_train, y_train)
    cv_scores = np.empty(2) # from optuna blog
    
    # splits the data into groups, uses one as validation, trains models and gets results then repeats
    for fold_id, (train_index, valid_index) in tqdm(enumerate(cv.split(X,y))): 
        #split
        X_tr = X.loc[train_index, features]
        X_val = X.loc[valid_index, features]
        y_tr = y.loc[train_index]
        y_val = y.loc[valid_index]
        
        
        # model, note inverse weighting please
        train_set = lgb.Dataset(X_tr, y_tr, categorical_feature = cats, weight = 1/np.power(y_tr, 2))
        val_set = lgb.Dataset(X_val, y_val, categorical_feature = cats, weight = 1/np.power(y_val, 2))
        # Note that train() will return a model from the best iteration.
        model = lgb.train(param_grid, train_set, valid_sets = [train_set, val_set], feval=RMSPEMetric(), verbose_eval=250)
        #model.fit(X_train=book_df,y_train = book_df[target],test_book_df,cats = cats)
        
        # predict
        y_pred = model.predict(X_val[features])
        y_true = y_val
        return (np.sqrt(np.mean(np.square((y_true - y_pred)/y_true))))
    
    
#optimization_function = partial(objective, X=x, y=y)
study = optuna.create_study(direction='minimize')
study.optimize(objective, n_trials=20)
print('Number of finished trials:', len(study.trials))
print('Best trial:', study.best_trial.params)
def fit_model(params, X_train, y_train, X_test, features=features, cats=[], era='stock_id', fold_type='kfold', n_fold = 2, seed=42):
    """
    fit model with cv (cross validation)
    """
    
    models = []
    oof_df = X_train[['time_id', 'stock_id', target]].copy()
    oof_df['pred'] = np.nan
    y_preds = np.zeros((len(X_test),))
    
    if fold_type == 'stratifiedshuffle':
        cv = model_selecion.StratifiedShuffleSplit(n_splits=n_fold, random_state=seed)
        kf = cv.split(X_train, X_train[era])
    elif fold_type == 'kfold':
        cv = model_selection.KFold(n_splits=n_fold, shuffle=True, random_state=seed)
        kf = cv.split(X_train, y_train)
    
    fi_df = pd.DataFrame()
    fi_df['features'] = features
    fi_df['importance'] = 0
    
    for fold_id, (train_index, valid_index) in tqdm(enumerate(kf)):
        #split
        X_tr = X_train.loc[train_index, features]
        X_val = X_train.loc[valid_index, features]
        y_tr = y_train.loc[train_index]
        y_val = y_train.loc[valid_index]
        
        
        # model, note inverse weighting please
        train_set = lgb.Dataset(X_tr, y_tr, categorical_feature = cats, weight = 1/np.power(y_tr, 2))
        val_set = lgb.Dataset(X_val, y_val, categorical_feature = cats, weight = 1/np.power(y_val, 2))
        # model training
        model = lgb.train(params, train_set, valid_sets = [train_set, val_set], feval=RMSPEMetric(), verbose_eval=250)
        
        # feature importance
        fi_df[f'importance_fold{fold_id}'] = model.feature_importance(importance_type='gain')
        fi_df['importance'] += fi_df[f'importance_fold{fold_id}'].values
        
        # save model
        joblib.dump(model, f'model_fold{fold_id}.pkl')
        logger.debug('model saved!')
        
        # predict
        oof_df['pred'].iloc[valid_index] = model.predict(X_val)
        y_pred = model.predict(X_test[features])
        y_preds += y_pred / n_fold
        models.append(model)
        
    return oof_df, y_preds, models, fi_df
# %%time

params = {
    'n_estimators' : 813,
    'objective': 'rmse',
    'boosting_type': 'dart',
    'max_depth': -1,
    'learning_rate': 0.1,
    'num_leaves': 511,
    'min_data_in_leaf': 617,
    'max_bin': 70,
    'subsample': 0.82,
    'subsample_freq': 7,
    'feature_fraction': 0.6,
    'lambda_l1': 0.5,
    'lambda_l2': 1,
    'min_gain_to_split': 0.009275035155177913,
    'seed': 46,
    'early_stopping_rounds': 75,
    'verbose': -1
}

if MODE == 'TRAIN':
    oof_df, y_preds, models, fi_df = fit_model(params,
                                              book_df,
                                              book_df[target],
                                              test_book_df,
                                              features=features,
                                              cats = cats,
                                              era = None,
                                              fold_type = 'kfold',
                                              n_fold = 2,
                                              seed = 46)<jupyter_output><empty_output><jupyter_text>Feature Importance and Feature Reduction<jupyter_code>Feature Importance
if MODE == 'TRAIN':
    fi_df = fi_df.sort_values(by='importance', ascending=False)
    fi_df.to_csv('feature_importance.csv', index=False)
    
    #fig, ax = plt.subplots(1, 1, figsize=(10, 40))
    #sns.barplot(x='importance', y='features', data=fi_df.iloc[:30], ax=ax)
    logger.info(fi_df[['features', 'importance']].iloc[:50].to_markdown())
reduced_features = (fi_df['features'].loc[:35])
reduced_features = reduced_features.to_list()
reduced_features
reduced_features = ['book_wap1_realized_volatility',
 'book_wap2_realized_volatility',
 'book_wap_mean_realized_volatility',
 'stock_id',
 '__book_wap2_realized_volatility____min___time',
 'book_bid_spread_realized_volatility',
 '__book_wap_mean_realized_volatility____mean___time',
 '__book_wap1_realized_volatility____min___time',
 'book_total_volume_realized_volatility',
 '__book_wap_mean_realized_volatility____min___time',
 '__book_wap2_realized_volatility____mean___time',
 '__price____std__',
 '__book_wap1_realized_volatility____mean___stock',
 '__book_wap1_realized_volatility____mean___time',
 '__book_wap_diff_realized_volatility____min___time',
 '__book_wap1_realized_volatility____max___time',
 '__book_wap_mean_realized_volatility____max___time',
 '__book_wap2_realized_volatility____max___time',
 '__price____sum__',
 '__seconds_in_bucket____sum__',
 '__seconds_in_bucket____mean__',
 '__order_count____sum__',
 'book_wap_diff_realized_volatility',
 '__order_count____std__',
 '__book_wap_mean_realized_volatility____mean___stock',
 'book_price_spread_realized_volatility',
 '__seconds_in_bucket____std__',
 '__book_wap1_realized_volatility____min___stock',
 '__price____min__',
 '__size____max__',
 '__order_count____mean__',
 '__price____max__',
 '__size____mean__',
 '__size____std__',
 '__size____min__',
 '__book_wap2_realized_volatility____mean___stock',
 '__price____mean__',
 '__order_count____max__']
# %%time

params_1 = {
    'n_estimators' : 813,
    'objective': 'rmse',
    'boosting_type': 'dart',
    'max_depth': -1,
    'learning_rate': 0.1,
    'num_leaves': 511,
    'min_data_in_leaf': 617,
    'max_bin': 70,
    'subsample': 0.82,
    'subsample_freq': 7,
    'feature_fraction': 0.6,
    'lambda_l1': 0.5,
    'lambda_l2': 1,
    #'min_gain_to_split': 0.009275035155177913,
    'seed': 46,
    'early_stopping_rounds': 50,
    'verbose': -1
}

if MODE == 'TRAIN':
    oof_df, y_preds, models, fi_df = fit_model(params_1,
                                              book_df,
                                              book_df[target],
                                              test_book_df,
                                              features=reduced_features,
                                              cats = cats,
                                              era = None,
                                              fold_type = 'kfold',
                                              n_fold = 3,
                                              seed = 46)
MODE == 'INFERENCE'
if MODE == 'INFERENCE':
    """
    used for inference kernel only
    """
    y_preds = np.zeros(len(test_book_df))
    files = glob.glob(f'{MODEL_DIR}/*model*.pkl')
    assert len(files) > 0
    for i, f in enumerate(files):
        model = joblib.load(f)
        y_preds += model.predict(test_book_df[features])
    y_preds /= (i+1)
    
test_book_df[target] = y_preds
# test
test_book_df[target] = y_preds

# save the submit file
sub = test_book_df[['row_id', target]]
sub.to_csv('submission.csv',index = False)

logger.info('submitted!')
sub.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12008577555396711, 'fraction_numerical': 0.02226079852956193, 'mean_word_length': 2.881886642885454, 'pattern_counts': {'":': 13, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '316247', 'license': 'Apache-2.0', 'n_tokens_mistral': 3535, 'n_tokens_neox': 3311, 'n_words': 576, 'path': 'Optiver Realized Volatility Prediction/optiver_realized_vol_prediction.ipynb', 'repo_name': 'lorenzoolguin/ML-Data-Science-Projects', 'size': '9793'}