<jupyter_start><jupyter_text>H2O.ai GPU Edition Machine Learning $-$ Multi-GPU GBM Demo  In this demo, we will train 16 gradient boosting models (aka GBMs) on the Higgs boson dataset, with the goal to predict whether a given event in the particle detector stems from an actual Higgs boson. The dataset is about 500MB in memory (2M rows, 29 cols, double precision floating-point values), so it fits onto the GPU memory. By using multiple GPUs, we are able to speed up this process significantly, and can train all 16 models in less than 2 minutes (DGX-1 with 8 GPUs) vs 30 minutes on a dual-Xeon server.  Import dependencies (This requires Linux Ubuntu 16.04 with CUDA 8)<jupyter_code>## First time only: Install H2O
#!pip install http://s3.amazonaws.com/h2o-deepwater/public/nightly/deepwater-h2o-230/h2o-3.11.0.230-py2.py3-none-any.whl
import os
import time
import sys
import h2o
import subprocess
from h2o.estimators import H2OXGBoostEstimator
from concurrent.futures import ProcessPoolExecutor

import numpy as np
import pandas as pd
import matplotlib.patches as mpatches
import matplotlib.pyplot as plt
import matplotlib.image as image
import seaborn as sns
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
sns.set_style("whitegrid")<jupyter_output><empty_output><jupyter_text>Start H2O and import the dataset (Here: Higgs Boson dataset: 2M rows, 29 cols)<jupyter_code>#path = "https://s3.amazonaws.com/h2o-public-test-data/bigdata/laptop/higgs_head_2M.csv" ### this works too
path = "/opt/higgs_head_2M.csv"
target = 0
drop_cols=[]
trainrows=1000000

h2o.init(max_mem_size="16G")
t0 = time.time()
df_hex = h2o.import_file(path)
t1 = time.time()
print("Parsed the dataset in %r seconds" % (t1-t0))
for c in drop_cols:
    df_hex = df_hex.drop(c, axis=1)
df_hex.refresh()

print(df_hex.shape)<jupyter_output><empty_output><jupyter_text>Define a method to train an H2O XGBoost Gradient Boosting Model in a separate process<jupyter_code>def runH2OXGBoost(arg):
    h2o.connect(ip='localhost',port=54321,verbose=False)
    depth, sample_rate, frame_id, trainrows, drop_cols, target, NGPUS, gpu = arg
    try:
        df_hex = h2o.get_frame(frame_id)
        import os
        trainhex = df_hex[:trainrows,:]
        trainhex[target] = trainhex[target].asfactor()
        learn_rate=0.01
        num_round =100
        param = {
              "ntrees":num_round
            , "max_depth":depth
            , "learn_rate":learn_rate
            , "sample_rate":sample_rate
            , "col_sample_rate_per_tree":sample_rate
            , "min_rows":5
            , "seed":12345
            , "tree_method":"exact"
            , "dmatrix_type":"dense"
            , "backend":("gpu" if NGPUS > 0 else "cpu")         
            , "gpu_id":gpu
            , "score_tree_interval":num_round
        }
        model = H2OXGBoostEstimator(**param)
        print("Training H2O XGBoost model with depth=", depth, "sample_rate=", sample_rate, ("on GPU" if NGPUS>0 else "on CPU"), gpu)
        model.train(x = list(trainhex.columns), y = target, training_frame = trainhex)
    except:
        print("Unexpected error:", sys.exc_info()[0])<jupyter_output><empty_output><jupyter_text>Train 16 H2O XGBoost models each on 8, 4, 2, 1 GPUs and on 0 GPUs (i.e, CPUs only)<jupyter_code>times = []
GPU_list = [8,4,2,1,0]
for NGPUS in GPU_list:
    
    Executor = ProcessPoolExecutor(max_workers=max(1,NGPUS))
    h2o.no_progress()

    t0 = time.time()
    print("Starting to train H2O XGBoost on %d GPUs" % NGPUS)
    gpu=0
    arg = []
    futures = []
    for depth in range(6,13,2): ## depth 6,8,10,12
        for sample_rate in [x/10. for x in range(7,11)]: ## sample_rate 0.7,0.8,0.9,1.0
            work = (depth,sample_rate,df_hex.frame_id,trainrows, drop_cols, target, NGPUS, gpu)
            futures.append(Executor.submit(runH2OXGBoost, work))
            if (NGPUS>0):
                gpu=(gpu+1) % NGPUS

    import concurrent.futures
    concurrent.futures.wait(futures)

    t1 = time.time()
    print("Time to train H2O on %d GPUs: %r\n\n" % (NGPUS, (t1-t0)))
    times.append(t1-t0)
print(times)<jupyter_output><empty_output><jupyter_text>Compute test set error for all 16 H2O XGBoost models<jupyter_code>testhex         = df_hex[trainrows:trainrows+1000000,:]
testhex[target] = testhex[target].asfactor()
print(testhex.shape)

models = [x[0] for x in h2o.ls().values if "XGBoost_model" in x[0]]
for m in models:
    try:
        model = h2o.get_model(m)
        perf = h2o.make_metrics(model.predict(testhex)[:,2], testhex[target])
        print(m, "sample rate:", model.params['sample_rate']['actual'], 
              "max_depth:", model.params['max_depth']['actual'], 
              "Test AUC:", perf.auc())
    except:
        continue<jupyter_output><empty_output><jupyter_text>Performance plot<jupyter_code>df = pd.DataFrame({'nGPUs':GPU_list, 't_train':times})
label_dic = {'0':'2 CPUs**', '1':'1 GPU*', '2':'2 GPUs*', '4':'4 GPUs*', '8':'8 GPUs*'}
colors = {'gpu':'limegreen', 'cpu':'dodgerblue'}

df['hardware'] = df['nGPUs'].apply(lambda x: str(x))
df = df.replace({'hardware':label_dic})



fig = plt.figure(figsize=(12,9))
ax =sns.barplot(x="t_train", y="hardware", data=df, orient = 'h', 
                 palette = list(df["nGPUs"].apply(lambda x:colors['gpu' if x>0 else 'cpu'])))

title_text = 'H2O.ai Machine Learning $-$ Gradient Boosting\nTime to Train 16 H2O XGBoost Models'
plt.title(title_text, fontsize=20, y=1.02)
plt.xlabel('Time [sec]', fontsize=20, labelpad = 12, fontweight = 'bold')
plt.ylabel('')

ax.yaxis.set_label_position('right')
ax.set_xlim([0,df.t_train.max()+100])
ax.tick_params(labelsize = 18)


for p in ax.patches:
    #print(p)
    x=p.get_bbox().get_points()[:,0]
    y=p.get_bbox().get_points()[:,1]
    ax.annotate('{:.0f}'.format(x[1]), (x[1]-0.035*df.t_train.max(), y.mean()), 
            ha='center', va='center', fontsize = 20, color = 'w', fontweight = 'bold')

footnote_text=\
"*NVIDIA Tesla P100 (DGX-1), **Dual Intel Xeon E5-2698 v4"\
"\nHiggs dataset (binary classification): 1M rows, 29 cols; max_depth: {6,8,10,12}, sample_rate: {0.7,0.8,0.9,1.0}"
plt.annotate(footnote_text, (0,0), (df.t_train.max()/2, -80), fontsize = 16, xycoords='axes fraction',
             textcoords='offset points', va='top', ha='right')

ax.invert_yaxis()
plt.tight_layout()

green_patch = mpatches.Patch(color='limegreen', label='GPU')
blue_patch = mpatches.Patch(color='dodgerblue', label='CPU')
ax.legend(handles = [blue_patch, green_patch], markerscale = 10, fontsize = 20, loc = 4)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14416197290302937, 'fraction_numerical': 0.03668747145684274, 'mean_word_length': 3.9213483146067416, 'pattern_counts': {'":': 12, '<': 25, '<?xml version=': 0, '>': 29, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '74563', 'license': 'Apache-2.0', 'n_tokens_mistral': 2578, 'n_tokens_neox': 2406, 'n_words': 561, 'path': 'gtc/Multi-GPU-H2O-GBM.ipynb', 'repo_name': 'h2oai/perf', 'size': '6569'}