<jupyter_start><jupyter_text>FE<jupyter_code>from sklearn.neighbors import KNeighborsClassifier

def apply_fe(df, ss=None, mm=None, knn_dict={}):
    cols = [c for c in df.columns if 'var_' in c]
    df_out = df.copy(deep=True)
    if ss:
        df_ss = ss.transform(df[cols])        
#         if not knn_dict:
#             for i in [50]:
#                 t = time.time()
#                 knn_dict[i] = KNeighborsClassifier(i, n_jobs=4).fit(df_ss, df['target'])
#                 print('Trained KNN{} in {:.2f} sec'.format(i, time.time()-t))
#         for name,knn in knn_dict.items():
#             t = time.time()
#             df_out['knn_{}'.format(name)] = knn.predict_proba(df_ss)[:,1]
#             print('Applied KNN{} in {:.2f} sec'.format(name, time.time()-t))
#         for i in range(1,5):
#             df_out['n_sigma{}'.format(i)] = (df_ss >= i).sum(axis=1)
#         df_out['sum_sigmas'] = df_ss.sum(axis=1)
        del df_ss
        
#     if mm:
#         df_mm = mm.transform(df[cols])
#         df_out['sum_mm'] = df_mm.sum(axis=1)        
#         del df_mm
    
    return df_out, knn_dict
from sklearn.preprocessing import RobustScaler, StandardScaler, MinMaxScaler
ss = StandardScaler().fit(df_trn.iloc[:,2:])
mm = MinMaxScaler().fit(df_trn.iloc[:,2:])

df_trn, knn = apply_fe(df_trn.iloc[:,:], ss=ss, mm=mm, knn_dict={})
df_tst, _ = apply_fe(df_tst, ss=ss, mm=mm, knn_dict=knn)
df_trn.head()<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>from sklearn.model_selection import StratifiedKFold, KFold, GroupKFold
from sklearn.base import clone, ClassifierMixin, RegressorMixin
import lightgbm as lgb
from sklearn.metrics import mean_squared_error, roc_auc_score
from sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor
import time


def train_single_model(clf_, X_, y_, random_state_=314, opt_parameters_={}, fit_params_={}):
    '''
    A wrapper to train a model with particular parameters
    '''
    c = clone(clf_)
    c.set_params(**opt_parameters_)
    c.set_params(random_state=random_state_)
    return c.fit(X_, y_, **fit_params_)

def train_model_in_CV(model, X, y, metric, metric_args={},
                            model_name='xmodel',
                            seed=31416, n=5,
                            opt_parameters_={}, fit_params_={},
                            verbose=True,
                            groups=None, y_eval=None,
                            mlf=None, mlf_metric_name=None
                     ):
    # the list of classifiers for voting ensable
    clfs = []
    # performance 
    perf_eval = {'score_i_oof': 0,
                 'score_i_ave': 0,
                 'score_i_std': 0,
                 'score_i': [],
                 'fit_time': []
                }
    # full-sample oof prediction
    y_full_oof = pd.Series(np.zeros(shape=(y.shape[0],)), 
                          index=y.index)
    
    sample_weight=None
    if 'sample_weight' in metric_args:
        sample_weight=metric_args['sample_weight']
        
    index_weight=None
    if 'index_weight' in metric_args:
        index_weight=metric_args['index_weight']
        del metric_args['index_weight']
        
    doSqrt=False
    if 'sqrt' in metric_args:
        doSqrt=True
        del metric_args['sqrt']

    if groups is None:
        cv = StratifiedKFold(n, shuffle=True, random_state=seed) #Stratified
    else:
        cv = GroupKFold(n)
    # The out-of-fold (oof) prediction for the k-1 sample in the outer CV loop
    y_oof = pd.Series(np.zeros(shape=(X.shape[0],)), 
                      index=X.index)
    scores = []
    clfs = []
    feature_importances = []

    for n_fold, (trn_idx, val_idx) in enumerate(cv.split(X, (y!=0).astype(np.int8), groups=groups)):
        X_trn, y_trn = X.iloc[trn_idx], y.iloc[trn_idx]
        X_val, y_val = X.iloc[val_idx], y.iloc[val_idx]
        
        if 'LGBMRanker' in type(model).__name__ and groups is not None:
            G_trn, G_val = groups.iloc[trn_idx], groups.iloc[val_idx]        

        if fit_params_:
            # use _stp data for early stopping
            fit_params_["eval_set"] = [(X_trn,y_trn), (X_val,y_val)]
            fit_params_['verbose'] = verbose
            if index_weight is not None:
                fit_params_["sample_weight"] = y_trn.index.map(index_weight).values
                fit_params_["eval_sample_weight"] = [None, y_val.index.map(index_weight).values]
            if 'LGBMRanker' in type(model).__name__ and groups is not None:
                fit_params_['group'] = G_trn.groupby(G_trn, sort=False).count()
                fit_params_['eval_group'] = [G_trn.groupby(G_trn, sort=False).count(),
                                             G_val.groupby(G_val, sort=False).count()]

        #display(y_trn.head())
        t = time.time()
        clf = train_single_model(model, X_trn, y_trn, 314+n_fold, opt_parameters_, fit_params_)
        perf_eval['fit_time'].append(time.time()-t)

        clfs.append(('{}{}'.format(model_name,n_fold), clf))
        # oof predictions
        if isinstance(clf, RegressorMixin):
            y_oof.iloc[val_idx] = clf.predict(X_val)
        elif isinstance(clf, ClassifierMixin):
            y_oof.iloc[val_idx] = clf.predict_proba(X_val)[:,1]
        else:
            y_oof.iloc[val_idx] = clf.predict(X_val)
        # prepare weights for evaluation
        if sample_weight is not None:
            metric_args['sample_weight'] = y_val.map(sample_weight)
        elif index_weight is not None:
            metric_args['sample_weight'] = y_val.index.map(index_weight).values
        # prepare target values
        y_true_tmp = y_val if 'LGBMRanker' not in type(model).__name__  and y_eval is None else y_eval.iloc[val_idx]
        y_pred_tmp = y_oof.iloc[val_idx] if y_eval is None else y_oof.iloc[val_idx]        
        #store evaluated metric
        metric_value = metric(y_true_tmp, y_pred_tmp, **metric_args)
        scores.append(metric_value)
        if mlf is not None:
            mlf.log_metric("{}_Fold{}".format(mlf_metric_name, n_fold), metric_value)
        #
        fi_tmp = pd.DataFrame()
        fi_tmp["feature"] = X.columns
        if hasattr(clf, 'feature_importances_'):
            fi_tmp["importance"] = clf.feature_importances_
        fi_tmp["fold"] = n_fold + 1
        feature_importances.append(fi_tmp)
        #cleanup
        del X_trn, y_trn, X_val, y_val, y_true_tmp, y_pred_tmp

    # Store performance info for this CV
    if sample_weight is not None:
        metric_args['sample_weight'] = y_oof.map(sample_weight)
    elif index_weight is not None:
        metric_args['sample_weight'] = y_oof.index.map(index_weight).values
    perf_eval['score_i_oof'] = metric(y, y_oof, **metric_args)
    perf_eval['score_i'] = scores
    
    if doSqrt:
        for k in perf_eval.keys():
            if 'score' in k:
                perf_eval[k] = np.sqrt(perf_eval[k])
        scores = np.sqrt(scores)
            
    perf_eval['score_i_ave'] = np.mean(scores)
    perf_eval['score_i_std'] = np.std(scores)

    return clfs, perf_eval, y_oof, pd.concat(feature_importances, axis=0)

def print_perf_clf(name, perf_eval):
    print('Performance of the model:')    
    print('Mean(Val) score inner {} Classifier: {:.4f}+-{:.4f}'.format(name, 
                                                                      perf_eval['score_i_ave'],
                                                                      perf_eval['score_i_std']
                                                                     ))
    print('Min/max scores on folds: {:.4f} / {:.4f}'.format(np.min(perf_eval['score_i']),
                                                            np.max(perf_eval['score_i'])))
    print('OOF score inner {} Classifier: {:.4f}'.format(name, perf_eval['score_i_oof']))
    print('Scores in individual folds: {}'.format(['{:.4f}'.format(x) for x in perf_eval['score_i']]))
def learning_rate_decay_power(current_iter):
    '''
    The function defines learning rate deay for LGBM
    '''
    base_learning_rate = 1e-1
    min_lr = 1e-2
    lr = base_learning_rate  * np.power(.95, current_iter)
    return lr if lr > min_lr else min_lr

def learning_rate_steps_5k(current_iter):
    '''
    The function defines learning rate deay for LGBM
    '''
    period = 5000
    return 1e-2 / (1 + current_iter//period)

learning_rate_decay_power(1000)
mdl_inputs = {
    'lgbm_base': (
        lgb.LGBMClassifier(
            max_depth=-1,
            min_child_samples=40,
            random_state=314,
            silent=True,
            metric='None',
            n_jobs=4,
            n_estimators=25000,
            importance_type='gain'),
        {
            'colsample_bytree': 0.02,
            'subsample': 0.05,
            'min_child_weight': 100.0,
            'min_child_samples': 100,
            'learning_rate': 0.005,
            'num_leaves': 25,
#             'class_weight': 'balanced'
        },
#         {
#             'colsample_bytree': 0.05,
#             'subsample': 0.05,
#             'min_child_weight': 100.0,
#             'min_child_samples': 100,
#             'learning_rate': 0.01,
#             'num_leaves': 15,
# #             'scale_pos_weight': 0.3,
# #             'reg_alpha': 0,
# #             'reg_lambda': 50,
# #             'class_weight': 'balanced'
#         },
        {
            "early_stopping_rounds": 1000,
            "eval_metric": 'auc',
#             'callbacks': [lgb.reset_parameter(learning_rate=learning_rate_steps_5k)],
#             'callbacks': [lgb.reset_parameter(learning_rate=learning_rate_decay_power)],
        },
        df_trn['target'],
        None),
#         'lgbm_rf': (lgb.LGBMClassifier(boosting_type='rf', reg_lambda=0, #min_child_samples=400, 
#                          max_depth=-1, random_state=314, silent=True, metric='None', 
#                          n_jobs=4, bagging_freq=1, importance_type='gain'),
#                {'colsample_bytree': 0.05, 'min_child_samples': 300, 'min_child_weight': 100, 
#                   'num_leaves': 100, 'subsample': 0.2, 'n_estimators':1500,}, 
#                {},
#                df_trn['target'],
#             None),
}

from scipy.stats import randint as sp_randint
from scipy.stats import uniform as sp_uniform
param_test = {
    'num_leaves': [7, 15, 25, 70],#sp_randint(6, 100),
#     'min_child_samples': [100,300,500,700,900],#sp_randint(100, 1000),
    'min_child_weight': [10,100],#[1e-5, 1e-3, 1e-2, 1e-1, 1, 1e1, 1e2, 1e3, 1e4],
    'subsample': [0.1, 0.15, 0.2, 0.4],#sp_uniform(loc=0.5, scale=0.5),
    'colsample_bytree': [0.02,0.05,0.1],#[0.35,0.50,0.65,0.75,0.85,0.95],#sp_uniform(loc=0.5, scale=0.5),
#     'reg_alpha': [0, 1e-1],
#     'reg_lambda': [0, 1e-3, 1e-1, 1]
}

from sklearn.model_selection import ParameterSampler, ParameterGrid
par_sampler = ParameterSampler(param_test, n_iter=2, random_state=31416)

par_grid = ParameterGrid([#{'colsample_bytree':[0.035,0.065]},
                          #{'subsample': [0.1, 0.15, 0.25, 0.3, 0.4]}
                          {'n_estimators': [4000, 8000]}
                         ])

features_not2use = ['ID_code', 'target']
# for v in ['var_89', 'var_139', 'var_12', 'var_81']:
#     kg.plot_var_for2classes(df_trn, v, bins=50,
#                             target_name='target')
#     plt.title(v)<jupyter_output><empty_output><jupyter_text>Hyper-parameter optimisation<jupyter_code>feats = [c for c in df_trn.columns if c not in features_not2use]

n_trn = None
seed_cv = 31416

mdls = {}
results = {}
y_oofs = {}
fis = {}  # feature importances

mlflow.set_experiment('TMP')

for name, (mdl, mdl_pars, fit_pars, y_, g_) in mdl_inputs.items():
    for ps in par_sampler: # par_sampler
        mdl_pars_ = copy.deepcopy(mdl_pars)
        mdl_pars_.update(ps)
        with mlflow.start_run(source_type=SourceType.NOTEBOOK, 
                              source_version=kg.get_last_git_commit()):
            print('--------------- {} -----------'.format(name))
            mlflow.set_tag('model_name', name)
            # Logging
            for k, v in mdl_pars_.items():
                mlflow.log_param(k, v)
            if n_trn is not None:
                mlflow.log_param('n_trn', n_trn)
            mlflow.log_param('seed_cv', seed_cv)

            mdl_, perf_eval_, y_oof_, fi_ = train_model_in_CV(
                mdl,
                df_trn[feats].iloc[:n_trn, :],
                y_.iloc[:n_trn],
                roc_auc_score,
                metric_args={},
                model_name=name,
                opt_parameters_=mdl_pars_,
                fit_params_=fit_pars,
                n=n_cv,
                seed=seed_cv,
                verbose=2000, groups=g_,
                mlf=mlflow, mlf_metric_name='AUC'
            )
            results[name] = perf_eval_
            mdls[name] = mdl_
            y_oofs[name] = y_oof_
            fis[name] = fi_
            print_perf_clf(name, perf_eval_)

            # metrics
            mlflow.log_metric("AUC", perf_eval_['score_i_ave'])
            mlflow.log_metric("AUC_STD", perf_eval_['score_i_std'])
#             for i in range(n_cv):
#                 m = mdls['lgbm_base'][i][1]
#                 for j in range(m.best_iteration_):
#                     mlflow.log_metric("AUC_Fold{}".format(i), m.evals_result_['valid_1']['auc'][j])
            fit_time_ave = np.mean(perf_eval_['fit_time'])
            if hasattr(mdl_[0][1],'n_estimators'):
                n_trees_ave = np.nanmean([m[1].n_estimators for m in mdl_])
            if getattr(mdl_[0][1], 'best_iteration_', None) is not None:
                n_trees_ave = np.nanmean([m[1].best_iteration_ for m in mdl_])    
            mlflow.log_metric('Time_sec', fit_time_ave)
            mlflow.log_metric('N_trees', n_trees_ave)
            mlflow.log_metric('Time_per_tree_msec', fit_time_ave / n_trees_ave * 1000)
            # artifacts
            # OOF
            y_oof_.to_csv('out/oof.csv', index=False)
            mlflow.log_artifact(os.getcwd()+'/out/oof.csv')
            # Submission
            sub = pd.DataFrame(index=df_tst['ID_code'])
            sub['target'] = 0
            for n, m in mdl_:
                sub['target'] += m.predict_proba(df_tst[feats])[:,1]/n_cv
            sub.to_csv('out/sub.csv', index=False)
            mlflow.log_artifact(os.getcwd()+'/out/sub.csv')
            #
            for v in fis:
                kg.display_importances(fis[v], n_feat=20, fout_name='out/fi.png')
                mlflow.log_artifact(os.getcwd()+'/out/fi.png')<jupyter_output>--------------- lgbm_base -----------
Training until validation scores don't improve for 1000 rounds.
[2000]	training's auc: 0.893961	valid_1's auc: 0.874072
[4000]	training's auc: 0.917147	valid_1's auc: 0.88836
[6000]	training's auc: 0.929889	valid_1's auc: 0.894898
[8000]	training's auc: 0.93826	valid_1's auc: 0.89824
[10000]	training's auc: 0.944614	valid_1's auc: 0.899907
[12000]	training's auc: 0.950071	valid_1's auc: 0.900662
[14000]	training's auc: 0.95505	valid_1's auc: 0.901063
[16000]	training's auc: 0.959597	valid_1's auc: 0.90119
[18000]	training's auc: 0.963753	valid_1's auc: 0.90127
[20000]	training's auc: 0.967584	valid_1's auc: 0.901323
[22000]	training's auc: 0.971056	valid_1's auc: 0.90134
Early stopping, best iteration is:
[21270]	training's auc: 0.969818	valid_1's auc: 0.901365
Training until validation scores don't improve for 1000 rounds.
[2000]	training's auc: 0.894929	valid_1's auc: 0.871847
[4000]	training's auc: 0.91747	valid_1's auc: 0.886087
[6000]	training[...]<jupyter_text>Single model run<jupyter_code>feats = [c for c in df_trn.columns if c not in features_not2use]

n_trn = None#-100000
n_ho = None
seed_cv = 31416

mdls = {}
results = {}
y_oofs = {}
fis = {}  # feature importances

mlflow.set_experiment('TMP')

for name, (mdl, mdl_pars, fit_pars, y_, g_) in mdl_inputs.items():
    mdl_pars_ = copy.deepcopy(mdl_pars)
    with mlflow.start_run(source_type=SourceType.NOTEBOOK,
                          source_version=kg.get_last_git_commit()):
        print('--------------- {} -----------'.format(name))
        mlflow.set_tag('model_name', name)
        # Logging
        for k, v in mdl_pars_.items():
            mlflow.log_param(k, v)
        if n_trn is not None:
            mlflow.log_param('n_trn', n_trn)
        mlflow.log_param('seed_cv', seed_cv)

        mdl_, perf_eval_, y_oof_, fi_ = train_model_in_CV(mdl,
            df_trn[feats].iloc[:n_trn, :], y_.iloc[:n_trn],
            roc_auc_score,
            metric_args={},
            model_name=name,
            opt_parameters_=mdl_pars_,
            fit_params_=fit_pars,
            n=n_cv, seed=seed_cv,
            verbose=500, groups=g_,
            mlf=mlflow, mlf_metric_name='AUC')
        results[name] = perf_eval_
        mdls[name] = mdl_
        y_oofs[name] = y_oof_
        fis[name] = fi_
        print_perf_clf(name, perf_eval_)

        # metrics
        mlflow.log_metric("AUC", perf_eval_['score_i_ave'])
        mlflow.log_metric("AUC_STD", perf_eval_['score_i_std'])
#             for i in range(n_cv):
#                 m = mdls['lgbm_base'][i][1]
#                 for j in range(m.best_iteration_):
#                     mlflow.log_metric("AUC_Fold{}".format(i), m.evals_result_['valid_1']['auc'][j])
        fit_time_ave = np.mean(perf_eval_['fit_time'])
        if hasattr(mdl_[0][1],'n_estimators'):
            n_trees_ave = np.nanmean([m[1].n_estimators for m in mdl_])
        if getattr(mdl_[0][1], 'best_iteration_', None) is not None:
            n_trees_ave = np.nanmean([m[1].best_iteration_ for m in mdl_])
        mlflow.log_metric('Time_sec', fit_time_ave)
        mlflow.log_metric('N_trees', n_trees_ave)
        mlflow.log_metric('Time_per_tree_sec', fit_time_ave / n_trees_ave * 1000)
#         # artifacts
#         # OOF
#         y_oof_.to_csv('out/oof.csv', index=False)
#         mlflow.log_artifact(os.getcwd()+'/out/oof.csv')
#         # Submission
#         sub = pd.DataFrame(index=df_tst['ID_code'])
#         sub['target'] = 0
#         for n, m in mdl_:
#             sub['target'] += m.predict_proba(df_tst[feats])[:,1]/n_cv
#         sub.to_csv('out/sub.csv', index=False)
#         mlflow.log_artifact(os.getcwd()+'/out/sub.csv')
#         #
#         kg.display_importances(fis['lgbm_base'], n_feat=20, fout_name='out/fi.png')
#         mlflow.log_artifact(os.getcwd()+'/out/fi.png')
# oof's for different targets
oof_0 = y_oofs['lgbm_base'][df_trn['target']==0]
oof_1 = y_oofs['lgbm_base'][df_trn['target']==1]
# ranked oof's for different targets
oof_rnk_0 = y_oofs['lgbm_base'].rank(pct=True)[df_trn['target']==0]
oof_rnk_1 = y_oofs['lgbm_base'].rank(pct=True)[df_trn['target']==1]
plt.figure(figsize=(12,6))
oof_0.plot.hist(bins=20, alpha=0.6)
oof_1.plot.hist(bins=20, alpha=0.6)
plt.yscale('log')
plt.figure(figsize=(12,6))
oof_rnk_0.plot.hist(bins=20, alpha=0.6)
oof_rnk_1.plot.hist(bins=20, alpha=0.6)
#plt.yscale('log')
oof_rnk_0.nlargest(10)
oof_rnk_1.nlargest(10)
df_trn.loc[oof_rnk_0.nlargest(10).index]
df_trn.loc[oof_rnk_0.nlargest(10).index]
oof_0.shape
oof_1.shape<jupyter_output><empty_output><jupyter_text>Importances  SHAP<jupyter_code>import shap
shap.initjs()
n_shap=10000
explainer = shap.TreeExplainer(mdls['lgbm_base'][0][1])
shap_values = explainer.shap_values(df_trn[feats].iloc[:n_shap,:])
shap.summary_plot(shap_values, df_trn[feats].iloc[:n_shap,:], plot_type="bar")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13918530679253027, 'fraction_numerical': 0.035450441206648885, 'mean_word_length': 2.3742426865154926, 'pattern_counts': {'":': 2, '<': 20, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '291735', 'license': 'MIT', 'n_tokens_mistral': 7609, 'n_tokens_neox': 7029, 'n_words': 1211, 'path': 'Process.ipynb', 'repo_name': 'mlisovyi/KaggleSantander2019', 'size': '19492'}