<jupyter_start><jupyter_text>LigthGBM - CLASSIFICATION - BAYESIAN OPTIMIZATION<jupyter_code>import numpy as np
from scipy.stats import uniform as sp_rand
from sklearn import datasets
import lightgbm as lgb
from lightgbm import LGBMClassifier
from bayes_opt import BayesianOptimization
import pandas as pd
from datetime import datetime
from time import time
from sklearn.metrics import roc_auc_score, auc
import warnings
warnings.filterwarnings('ignore')<jupyter_output><empty_output><jupyter_text>GLOBAL VARIABLES<jupyter_code>DATAPATH = 'data/train_test/'
SEED = 47
NITER = 20
CV = 3
SCORE = 'roc_auc'
handlingnull = False
NJOBS = -1
USEGPU = False
NCLASS = 3 # number class to predict (if bivar set 0)<jupyter_output><empty_output><jupyter_text>LOAD DATASET<jupyter_code>train_features = X = pd.read_pickle('data/processed/X.pkl')
train_labels = pd.read_pickle('data/processed/y.pkl')
y_train = train_labels.copy()
train_labels = train_labels['target']
campaign = pd.read_pickle('data/features/campaign_quarter_001.pkl')
digital = pd.read_pickle('data/features/digital_features_period_001.pkl')
rcc = pd.read_pickle('data/features/X_rcc_features_ten_001.pkl')
rcc.drop(['id_persona', 'codmes'], inplace=True, axis=1)
train_features = train_features.join(campaign).join(rcc).join(digital)
# create dataset for lightgbm
lgb_train = lgb.Dataset(train_features.values, train_labels.values)<jupyter_output><empty_output><jupyter_text>TRAIN MODEL  Set Search hyperparameters<jupyter_code># ======== General Parameters ======= #

# Select the type of model to run at each iteration. gbtree or gblinear.
boosting = 'gbdt'


# ======== Booster Parameters ======== # 

# Analogous to learning rate in GBM. 
# Typical final values to be used: 0.01-0.2
eta = [0.01] 


# A node is split only when the resulting split gives a positive reduction in the loss function. 
# Gamma specifies the minimum loss reduction required to make a split.
gamma = [i/10.0 for i in range(0,5)]


# Control the balance of positive and negative weights, useful for unbalanced classes. 
# A typical value to consider: sum(negative instances) / sum(positive instances)scale_pos_weight = 1
scale_pos_weight = (len(train_labels) - sum(train_labels))/sum(train_labels)


# Learning Task Parameters
# This defines the loss function to be minimized. See documentation
# -  options: regression, regression_l1, huber, fair, poisson, quantile, 
# mape, gamma, tweedie, binary, multiclass, multiclassova, cross_entropy, cross_entropy_lambda,
# lambdarank, aliases: objective_type, app, application
objective  = 'binary'


# The metric to be used for validation data.
# - rmse, root square loss, aliases: root_mean_squared_error, l2_root
# - quantile, Quantile regression
# - mape, MAPE loss, aliases: mean_absolute_percentage_error
# - huber, Huber loss
# - fair, Fair loss
# - poisson, negative log-likelihood for Poisson regression
# - gamma, negative log-likelihood for Gamma regression
# - gamma_deviance, residual deviance for Gamma regression
# - tweedie, negative log-likelihood for Tweedie regression
# - ndcg, NDCG, aliases: lambdarank
# - map, MAP, aliases: mean_average_precision
# - auc, AUC
# - binary_logloss, log loss, aliases: binary
metric = 'auc'<jupyter_output><empty_output><jupyter_text>[lightGBM params](https://lightgbm.readthedocs.io/en/latest/Parameters.html) help(lgb.LGBMClassifier)  Define the search space<jupyter_code>pds = {
    # Minimal number of data in one leaf. Can be used to deal with over-fitting
    # default = 20, type = int, aliases: min_data_per_leaf, min_data, min_child_samples, 
    'num_leaves': (20, 100),

    # Denotes the fraction of columns to be randomly samples for each tree.
    'feature_fraction': (0.1, 0.9),
    
    # Denotes the fraction of observations to be randomly samples for each tree.
    'bagging_fraction': (0.8, 1),

    # The maximum depth of a tree
    'max_depth': (9, 13 ),

    'min_split_gain': (0.001, 0.1),

    # Minimal sum hessian in one leaf. Like min_data_in_leaf, it can be used to deal with over-fitting
    # Default 1e-3
    'min_child_weight': (30, 50),    
}
pds<jupyter_output><empty_output><jupyter_text>Find num boost<jupyter_code>early_stopping_rounds = 50
# specify your configurations as a dict
lgb_params = {
    'boosting_type': boosting,
    'objective': objective,
    'metric': metric,
    'num_leaves': 31,
    'learning_rate': 0.05,
    'feature_fraction': 0.9,
    'bagging_fraction': 0.8,
    'bagging_freq': 5,
    'num_threads' : 8,
    'verbose': 0,
    #'num_class':  NCLASS,
    'seed' : SEED
}
cvresult = lgb.cv(lgb_params, lgb_train, num_boost_round = 1000, nfold = CV, metrics = metric, early_stopping_rounds = early_stopping_rounds, seed = SEED)
n_estimators = pd.DataFrame(cvresult).shape[0]
print("Best number of estimators found: ", n_estimators)<jupyter_output>Best number of estimators found:  364<jupyter_text>Bayesian optimization hyperparameters<jupyter_code>test_probs = []
test_auc = []

def hyp_lgbm(num_leaves, feature_fraction, bagging_fraction, max_depth, min_split_gain, min_child_weight):
      
    params = {'boosting_type': boosting,
              'objective': objective,
              'num_iterations': n_estimators,
              'learning_rate':eta, 
              #'early_stopping_round':50,
              'metric': metric,
              'seed':SEED} # Default parameters
    
    
    params["num_leaves"] = int(round(num_leaves))
    params['feature_fraction'] = max(min(feature_fraction, 1), 0)
    params['bagging_fraction'] = max(min(bagging_fraction, 1), 0)
    params['max_depth'] = int(round(max_depth))
    params['min_split_gain'] = min_split_gain
    params['min_child_weight'] = min_child_weight
    params['min_split_gain'] = min_split_gain
    
    
    model = LGBMClassifier().set_params(**params)
    
    drop_cols = ["codmes"]
    
    
    for m in train_features.codmes.unique():
        Xt = train_features[train_features.codmes != m]
        yt = train_labels[train_labels.index.isin(Xt.index)]
        Xt = Xt.drop(drop_cols, axis=1)

        Xv = train_features[train_features.codmes == m]
        yv = train_labels[train_labels.index.isin(Xv.index)]
    
        
        res = model.fit(Xt, yt,  eval_metric=metric, verbose=False)
        
        pred = model.predict_proba(Xv.drop(drop_cols, axis=1))[:, -1]
    
        test_auc.append(roc_auc_score(yv.values, pred))
        
        test_probs.append(pd.Series(pred,
                                index=Xv.index, name="probs"))
        
        
    
    # Extract the best score
    best_score = np.mean(test_auc)
    
    # Loss must be minimized
    #loss = 1 - best_score

    return best_score
# Surrogate model
optimizer = BayesianOptimization(hyp_lgbm,pds,random_state=7)
# Optimize
optimizer.maximize(init_points=5, n_iter=NITER)<jupyter_output>|   iter    |  target   | baggin... | featur... | max_depth | min_ch... | min_sp... | num_le... |
-------------------------------------------------------------------------------------------------
| [0m 1       [0m | [0m 0.7929  [0m | [0m 0.8153  [0m | [0m 0.7239  [0m | [0m 10.75   [0m | [0m 44.47   [0m | [0m 0.09782 [0m | [0m 63.08   [0m |
| [0m 2       [0m | [0m 0.7916  [0m | [0m 0.9002  [0m | [0m 0.1576  [0m | [0m 10.07   [0m | [0m 40.0    [0m | [0m 0.06824 [0m | [0m 84.3    [0m |
| [0m 3       [0m | [0m 0.7906  [0m | [0m 0.8762  [0m | [0m 0.1527  [0m | [0m 10.15   [0m | [0m 48.19   [0m | [0m 0.02213 [0m | [0m 56.17   [0m |
| [0m 5       [0m | [0m 0.7898  [0m | [0m 0.9818  [0m | [0m 0.2065  [0m | [0m 11.09   [0m | [0m 45.01   [0m | [0m 0.06723 [0m | [0m 57.42   [0m |
| [0m 6       [0m | [0m 0.7904  [0m | [0m 0.9684  [0m | [0m 0.5078  [0m | [0m 9.112   [0m | [0m 30.15   [0m | [0m 0.03288 [0m | [0m 99.99   [...]<jupyter_text>Best parameter<jupyter_code>optimizer.max['params']<jupyter_output><empty_output><jupyter_text>Saving best hyperparameters<jupyter_code>best_params = optimizer.max['params']
best_params['n_estimators'] = n_estimators
best_params
np.save('output/hyperparameters/bayesianopt_gbm_classifier_bestparams_d' + str(datetime.now().date()) + '.npy', best_params)<jupyter_output><empty_output><jupyter_text>Find best cutoff best<jupyter_code>from scipy.optimize import differential_evolution

res = y_train.join(pd.concat(test_probs).rename("probs"))
optimization = differential_evolution(lambda c: -((res.probs > c[0]) * res.margen / res.margen.sum()).sum(), [(0, 1)])
optimization
cutoff = 0.04843624
def plot_roc_curve(true_labels, scores):
    ### actual code for roc + threshold charts start here 
    # compute fpr, tpr, thresholds and roc_auc
    fpr, tpr, thresholds = roc_curve(true_labels, scores)
    roc_auc = auc(fpr, tpr) # compute area under the curve

    plt.figure()
    plt.plot(fpr, tpr, label='ROC curve (area = %0.2f)' % (roc_auc))
    plt.plot([0, 1], [0, 1], 'k--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Receiver operating characteristic')
    plt.legend(loc="lower right")

    # create the axis of thresholds (scores)
    ax2 = plt.gca().twinx()
    ax2.plot(fpr, thresholds, markeredgecolor='r',linestyle='dashed', color='r')
    ax2.set_ylabel('Threshold',color='r')
    ax2.set_ylim([thresholds[-1],thresholds[0]])
    ax2.set_xlim([fpr[0],fpr[-1]])

    plt.show()
    
    pass
from sklearn.metrics import roc_auc_score, roc_curve
%matplotlib inline
import matplotlib.pyplot as plt
plot_roc_curve(res['target'], res['probs'])
# Find prediction to the dataframe applying threshold
res['pred'] = np.where(res.probs > cutoff,1,0)
# Print confusion Matrix
confusion_matrix(res['target'], res['pred'])
res.shape
from sklearn.metrics import confusion_matrix<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14203104392817287, 'fraction_numerical': 0.0387541848432586, 'mean_word_length': 3.8205378973105133, 'pattern_counts': {'":': 0, '<': 39, '<?xml version=': 0, '>': 41, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '567009', 'license': 'RSA-MD', 'n_tokens_mistral': 3920, 'n_tokens_neox': 3523, 'n_words': 747, 'path': 'notebooks/models/light_gbm/bayesianopt_lgb_classifier.ipynb', 'repo_name': 'Guillin/honeycomb', 'size': '9857'}