<jupyter_start><jupyter_text>Baseline Lightgbm<jupyter_code>lgbm_params = {
        'boosting_type': 'gbdt',
        'objective': 'regression',
        'nthread': -1,
        'num_leaves': 2 ** 4,
        'learning_rate': 0.05,
        'max_depth': -1,
#         'max_bin': 255,
#         'subsample_for_bin': 50000,
        'subsample': 0.8,
        'subsample_freq': 1,
        'colsample_bytree': 0.6,
        'reg_alpha': 1,
        'reg_lambda': 0,
        'min_split_gain': 0.5,
        'min_child_weight': 1,
        'min_child_samples': 10,
        'scale_pos_weight': 1,
        'early_stopping_round': 20,
        'metric': 'rmsle',
        'verbose': 0
    }
cv_results = lgbm.cv(lgbm_params,
                     d_train,
                     num_boost_round=50,
                     nfold=3,
                     feval=lgb_rmsle_score,
                     early_stopping_rounds=300,
                     verbose_eval=50)<jupyter_output>[50]	cv_agg's rmsle: 0.427718 + 0.00978353<jupyter_text>Param Fine Tuning<jupyter_code>def fine_tune_param_cv(lgbm_params, param, values):
    print lgbm_params
    print '===> fine tuning {}...'.format(param)
    min_test_rmse_mean = 100
    best_value = lgbm_params[param]
    for value in values:
        print 'fine tuning {} = {}'.format(param, value)
        lgbm_params[param] = value
        cv_results = lgbm.cv(lgbm_params,
                     d_train,
                     num_boost_round=50,
                     nfold=3,
                     feval=lgb_rmsle_score,
                     early_stopping_rounds=300,
                     verbose_eval=50)
        
        test_rmse_mean = cv_results['rmsle-mean'][-1]
        print "CV RMSE : test_rmse_mean = %.7g" % (test_rmse_mean)
        if test_rmse_mean < min_test_rmse_mean:
            min_test_rmse_mean = test_rmse_mean
            best_value = value
    
    print 'best {} = {}, min_test_rmse_mean = {}'.format(param, best_value, min_test_rmse_mean)
    return lgbm_params, best_value
lgbm_params = {
        'boosting_type': 'gbdt',
        'objective': 'regression',
        'nthread': -1,
    
        'num_leaves': 2 ** 6,
        'learning_rate': 0.05,
        'max_depth': 10,
#         'max_bin': 255,
#         'subsample_for_bin': 50000,
        'subsample': 0.8,
        'subsample_freq': 1,
        'colsample_bytree': 0.6,
        'reg_alpha': 1,
        'reg_lambda': 0,
        'min_split_gain': 0.5,
        'min_child_weight': 1,
        'min_child_samples': 10,
        'scale_pos_weight': 1,
        'early_stopping_round': 20,
        'metric': 'rmsle',
        'verbose': 0
    }
_ = fine_tune_param_cv(lgbm_params, 'subsample', common_num_range(0.6, 0.76, 0.05))
lgbm_params = {
        'boosting_type': 'gbdt',
        'objective': 'regression',
        'nthread': -1,
        'subsample': 0.75,
        'subsample_freq': 1,
    
        'num_leaves': 2 ** 6,
        'learning_rate': 0.05,
        'max_depth': 10,
#         'max_bin': 255,
#         'subsample_for_bin': 50000,
        
        'colsample_bytree': 0.6,
        'reg_alpha': 1,
        'reg_lambda': 0,
        'min_split_gain': 0.5,
        'min_child_weight': 1,
        'min_child_samples': 10,
        'scale_pos_weight': 1,
        'early_stopping_round': 20,
        'metric': 'rmsle',
        'verbose': 0
    }
_ = fine_tune_param_cv(lgbm_params, 'colsample_bytree', common_num_range(0.6, 0.7, 0.05))
lgbm_params = {
        'boosting_type': 'gbdt',
        'objective': 'regression',
        'nthread': -1,
        'subsample': 0.75,
        'subsample_freq': 1,
        'colsample_bytree': 0.6,
    
        'num_leaves': 2 ** 6,
        'learning_rate': 0.05,
        'max_depth': 10,
#         'max_bin': 255,
#         'subsample_for_bin': 50000,
        
        'reg_alpha': 1,
        'reg_lambda': 0,
        'min_split_gain': 0.5,
        'min_child_weight': 1,
        'min_child_samples': 10,
        'scale_pos_weight': 1,
        'early_stopping_round': 20,
        'metric': 'rmsle',
        'verbose': 0
    }
_ = fine_tune_param_cv(lgbm_params, 'max_depth', [8,10,12,14])
lgbm_params = {
        'boosting_type': 'gbdt',
        'objective': 'regression',
        'nthread': -1,
        'subsample': 0.75,
        'subsample_freq': 1,
        'colsample_bytree': 0.6,
    
        'num_leaves': 2 ** 6,
        'learning_rate': 0.05,
        'max_depth': 10,
#         'max_bin': 255,
#         'subsample_for_bin': 50000,
        
        'reg_alpha': 1,
        'reg_lambda': 0,
        'min_split_gain': 0.5,
        'min_child_weight': 1,
        'min_child_samples': 10,
        'scale_pos_weight': 1,
        'early_stopping_round': 20,
        'metric': 'rmsle',
        'verbose': 0
    }
_ = fine_tune_param_cv(lgbm_params, 'num_leaves', [2 ** 6, 2 ** 7, 2 ** 8, 2 ** 9])
lgbm_params = {
        'boosting_type': 'gbdt',
        'objective': 'regression',
        'nthread': -1,
        'subsample': 0.75,
        'subsample_freq': 1,
        'colsample_bytree': 0.6,
    
        'num_leaves': 2 ** 6,
        'learning_rate': 0.05,
        'max_depth': 14,
#         'max_bin': 255,
#         'subsample_for_bin': 50000,
        
        'reg_alpha': 1,
        'reg_lambda': 0,
        'min_split_gain': 0.5,
        'min_child_weight': 1,
        'min_child_samples': 10,
        'scale_pos_weight': 1,
        'early_stopping_round': 20,
        'metric': 'rmsle',
        'verbose': 0
    }
_ = fine_tune_param_cv(lgbm_params, 'num_leaves', [2 ** 6, 2 ** 7, 2 ** 8, 2 ** 9])
lgbm_params = {
        'boosting_type': 'gbdt',
        'objective': 'regression',
        'nthread': -1,
        'subsample': 0.75,
        'subsample_freq': 1,
        'colsample_bytree': 0.6,
    
        'num_leaves': 2 ** 6,
        'learning_rate': 0.05,
        'max_depth': 10,
        
        'reg_alpha': 1,
        'reg_lambda': 0,
        'min_split_gain': 0.5,
        'min_child_weight': 1,
        'min_child_samples': 10,
        'scale_pos_weight': 1,
        'early_stopping_round': 20,
        'metric': 'rmsle',
        'verbose': 0
    }
_ = fine_tune_param_cv(lgbm_params, 'min_split_gain', [0.3, 0.4])
lgbm_params = {
        'boosting_type': 'gbdt',
        'objective': 'regression',
        'nthread': -1,
        'subsample': 0.75,
        'subsample_freq': 1,
        'colsample_bytree': 0.6,
        'min_split_gain': 0.4,
    
        'num_leaves': 2 ** 6,
        'learning_rate': 0.05,
        'max_depth': 10,
        
        'reg_alpha': 0.1,
        'reg_lambda': 0,
        
        'scale_pos_weight': 1,
        'early_stopping_round': 20,
        'metric': 'rmsle',
        'verbose': 0
    }
_ = fine_tune_param_cv(lgbm_params, 'reg_lambda', [0, 0.01, 0.1])<jupyter_output>{'num_leaves': 64, 'reg_alpha': 0.1, 'subsample_freq': 1, 'colsample_bytree': 0.6, 'scale_pos_weight': 1, 'learning_rate': 0.05, 'nthread': -1, 'min_split_gain': 0.4, 'subsample': 0.75, 'reg_lambda': 10, 'objective': 'regression', 'verbose': 0, 'max_depth': 10, 'metric': 'rmsle', 'boosting_type': 'gbdt'}
===> fine tuning reg_lambda...
fine tuning reg_lambda = 0
[50]	cv_agg's rmsle: 0.414976 + 0.0106041
CV RMSE : test_rmse_mean = 0.4149757
fine tuning reg_lambda = 0.01
[50]	cv_agg's rmsle: 0.415468 + 0.0104646
CV RMSE : test_rmse_mean = 0.4154676
fine tuning reg_lambda = 0.1
[50]	cv_agg's rmsle: 0.414419 + 0.0104
CV RMSE : test_rmse_mean = 0.4144187
best reg_lambda = 0.1, min_test_rmse_mean = 0.414418724084
{'fraction_non_alphanumeric': 0.14787505027483577, 'fraction_numerical': 0.05965947177905886, 'mean_word_length': 2.0712227254014, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '612755', 'license': 'MIT', 'n_tokens_mistral': 3144, 'n_tokens_neox': 2788, 'n_words': 513, 'path': 'model/Lightgbm_Fine_Tuning.ipynb', 'repo_name': 'SunnyMarkLiu/Kaggle_NYC_Taxi_Trip_Duration', 'size': '7459'}