<jupyter_start><jupyter_text>Experiment 06: HIGGS boson This experiment uses the data from the [HIGGS dataset](https://archive.ics.uci.edu/ml/datasets/HIGGS) to predict the appearance of the Higgs boson. The dataset consists of 11 million of observations. More information about the data can be found in [loaders.py](libs/loaders.py).  For details of virtual machine we used and the versions of LightGBM and XGBoost, please refer to [experiment 1](01_airline.ipynb).<jupyter_code>import json
import sys
import matplotlib.pyplot as plt
import pkg_resources
from libs.loaders import load_higgs
from libs.timer import Timer
from libs.utils import get_number_processors
from sklearn.metrics import roc_auc_score, accuracy_score, precision_score, recall_score, f1_score
from sklearn.model_selection import train_test_split
from xgboost import XGBClassifier
from lightgbm import LGBMClassifier
import warnings
warnings.filterwarnings('ignore')

print("System version: {}".format(sys.version))
print("XGBoost version: {}".format(pkg_resources.get_distribution('xgboost').version))
print("LightGBM version: {}".format(pkg_resources.get_distribution('lightgbm').version))
%%time
df = load_higgs()
print(df.shape)
df.head()
num_rounds = 200
number_processors = get_number_processors()
print(number_processors)
xgb_clf_pipeline = XGBClassifier(max_depth=5, 
                                learning_rate=0.1, 
                                scale_pos_weight=2,
                                n_estimators=num_rounds,
                                gamma=0.1,
                                min_child_weight=1,
                                reg_lambda=1,
                                subsample=1,
                                nthread=number_processors
                                )
xgb_hist_clf_pipeline = XGBClassifier(max_depth=0, 
                                     learning_rate=0.1, 
                                     scale_pos_weight=2,
                                     n_estimators=num_rounds,
                                     gamma=0.1,
                                     min_child_weight=1,
                                     reg_lambda=1,
                                     subsample=1,
                                     max_leaves=2**5,
                                     grow_policy='lossguide',
                                     tree_method='hist',
                                     nthread=number_processors
                                     )
lgbm_clf_pipeline = LGBMClassifier(num_leaves=2**5, 
                                  learning_rate=0.1, 
                                  scale_pos_weight=2,
                                  n_estimators=num_rounds,
                                  min_split_gain=0.1,
                                  min_child_weight=1,
                                  reg_lambda=1,
                                  subsample=1,
                                  nthread=number_processors
                                  )
metrics_dict = {
    'Accuracy': accuracy_score,
    'Precision': precision_score,
    'Recall': recall_score,
    'AUC': roc_auc_score,
    'F1': f1_score,
}

def classification_metrics(metrics, y_true, y_pred):
    return {metric_name:metric(y_true, y_pred) for metric_name, metric in metrics.items()}
def generate_feables(df):
    X = df[df.columns.difference(['boson'])]
    y = df['boson']
    return X,y
X, y = generate_feables(df)
X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=77, test_size=500000)
results_dict = dict()<jupyter_output><empty_output><jupyter_text>XGBoost<jupyter_code>with Timer() as train_t:
    xgb_clf_pipeline.fit(X_train,y_train)
    
with Timer() as test_t:
    y_pred = xgb_clf_pipeline.predict(X_test)
results_dict['xgb']={
    'train_time': train_t.interval,
    'test_time': test_t.interval,
    'performance': classification_metrics(metrics_dict, 
                                          y_test, 
                                          y_pred) 
}
with Timer() as t_train:
    xgb_hist_clf_pipeline.fit(X_train,y_train)
with Timer() as t_test:
    y_pred = xgb_hist_clf_pipeline.predict(X_test)
results_dict['xgb_hist']={
    'train_time': t_train.interval,
    'test_time': t_test.interval,
    'performance': classification_metrics(metrics_dict, 
                                          y_test, 
                                          y_pred) 
}<jupyter_output><empty_output><jupyter_text>LightGBM<jupyter_code>with Timer() as train_t:
    lgbm_clf_pipeline.fit(X_train, y_train)
    
with Timer() as test_t:
    y_pred = lgbm_clf_pipeline.predict(X_test)
results_dict['lgbm']={
    'train_time': train_t.interval,
    'test_time': test_t.interval,
    'performance': classification_metrics(metrics_dict, 
                                          y_test, 
                                          y_pred) 
}
# Results
print(json.dumps(results_dict, indent=4, sort_keys=True))<jupyter_output>{
    "lgbm": {
        "performance": {
            "AUC": 0.694682949690134,
            "Accuracy": 0.707758,
            "F1": 0.7680747894958216,
            "Precision": 0.6627597069095391,
            "Recall": 0.9131831219806763
        },
        "test_time": 0.7120589099995414,
        "train_time": 119.34003880199998
    },
    "xgb": {
        "performance": {
            "AUC": 0.6859901403358623,
            "Accuracy": 0.699694,
            "F1": 0.7635493812093622,
            "Precision": 0.6551156676187414,
            "Recall": 0.9149984903381643
        },
        "test_time": 0.55617916600022,
        "train_time": 2996.1667750769993
    },
    "xgb_hist": {
        "performance": {
            "AUC": 0.6941216899970567,
            "Accuracy": 0.70721,
            "F1": 0.767674555527519,
            "Precision": 0.6623426413523601,
            "Recall": 0.9128434480676328
        },
        "test_time": 0.6464068210007099,
        "train_time": 121.21175534400027[...]
{'fraction_non_alphanumeric': 0.09190992493744787, 'fraction_numerical': 0.06288573811509592, 'mean_word_length': 1.809746954076851, 'pattern_counts': {'":': 27, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '249702', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2090, 'n_tokens_neox': 1797, 'n_words': 311, 'path': '3rdparty/fast_retraining/experiments/06_HIGGS.ipynb', 'repo_name': 'JohnZed/gbm-bench', 'size': '5995'}