<jupyter_start><jupyter_text>H2O<jupyter_code># NOTE: here onwards this is the first iteration for the stacked ensemble

import pandas as pd
import h2o
h2o.init(min_mem_size='12G')


mono_resistance_df_filledna = pd.read_csv("../data/processed/mono_resistance_df_filledna.csv")

mono_resistance_df_filledna.head()
mono_resistance_df_filledna= mono_resistance_df_filledna.set_index('SampleID')

mono_resistance_df_filledna.head()

# SMALL
# mono_resistance_df = pd.read_csv("C:/Users/abhi1/Projects/code/drug-resistance-prediction-cambiohack/src/data/processed/mono_resistance_df_small.csv", index_col='SampleID')
# mono_resistance_df.head()
mono_resistance_frame = h2o.H2OFrame(mono_resistance_df_filledna)
mono_resistance_frame.head()

# h2o.export_file(mono_resistance_frame, "../data/processed/mono_resistance_frame.csv")

mono_resistance_frame = h2o.import_file("../data/processed/mono_resistance_frame.csv")

mono_resistance_frame.head()
train, test = mono_resistance_frame.split_frame(ratios=[.70], seed=1234)

test.head()

# Import a sample binary outcome train/test set into H2O
train_higgs = h2o.import_file("../data/external/higgs_train_10k.csv")
test_higgs = h2o.import_file("../data/external/higgs_test_5k.csv")

train = train_higgs
train_higgs.head()
test = test_higgs
test_higgs.head()

# Identify predictors and response
x = train.columns
# y = "isResistant"
y = "response"
x.remove(y)
x[:10]


y

# For binary classification, response should be a factor
train[y] = train[y].asfactor()
test[y] = test[y].asfactor()

# for x in train.columns:
#     train[x] = train[x].asfactor()
#     test[x] = test[x].asfactor()
# Number of CV folds (to generate level-one data for stacking)
nfolds = 5

# There are a few ways to assemble a list of models to stack together:
# 1. Train individual models and put them in a list
# 2. Train a grid of models
# 3. Train several grids of models
# Note: All base models must have the same cross-validation folds and
# the cross-validated predicted values must be kept.


# 1. Generate a 2-model ensemble (GBM + RF)

from h2o.estimators.gbm import H2OGradientBoostingEstimator

# Train and cross-validate a GBM
my_gbm = H2OGradientBoostingEstimator(distribution="bernoulli",
                                      ntrees=10,
                                      max_depth=3,
                                      min_rows=2,
                                      learn_rate=0.2,
                                      nfolds=nfolds,
                                      fold_assignment="random",
                                      keep_cross_validation_predictions=True,
                                      seed=1234)

my_gbm.train(x=x, y=y, training_frame=train)

# save the model
# model_path = h2o.save_model(model= my_gbm, path="../data/processed/my_gbm_model", force=True)

# print(model_path)
model_path = "../data/processed/my_gbm_model\GBM_model_python_1601519217841_1"

# load the model
my_gbm = h2o.load_model(model_path)

# download the model built above to your local machine
# my_local_model = h2o.download_model(my_gbm, path="../data/processed/my_gbm_model/")

# upload the model that you just downloded above
# to the H2O cluster
# uploaded_model = h2o.upload_model(my_local_model)
# Generate predictions on a test set (if neccessary)
pred = saved_model.predict(test)

pred
from h2o.estimators import H2ONaiveBayesEstimator

# Train and cross-validate a NB
my_nb = H2ONaiveBayesEstimator(
                                 nfolds=nfolds, 
                                     fold_assignment = "random",
                               keep_cross_validation_predictions = True,
                                 seed=1234)

my_nb.train(x=x, y=y, training_frame=train)

# save the model
# model_path = h2o.save_model(model= my_nb, path="../data/processed/my_nb_model", force=True)

# print(model_path)
model_path = "../data/processed/my_nb_model/NaiveBayes_model_python_1601520350416_902"

# load the model
my_nb = h2o.load_model(model_path)

# download the model built above to your local machine
# my_local_model = h2o.download_model(my_gbm, path="../data/processed/my_gbm_model/")

# upload the model that you just downloded above
# to the H2O cluster
# uploaded_model = h2o.upload_model(my_local_model)
# Generate predictions on a test set (if neccessary)
pred = saved_model.predict(test)

pred
from h2o.estimators.random_forest import H2ORandomForestEstimator


# Train and cross-validate a RF
my_rf = H2ORandomForestEstimator(ntrees=50,
                                 nfolds=nfolds, 
                                 fold_assignment="random",
                                 keep_cross_validation_predictions=True,
                                 seed=1234)

my_rf.train(x=x, y=y, training_frame=train)

# save the model
# model_path = h2o.save_model(model= my_rf, path="../data/processed/my_rf_model", force=True)

# print(model_path)
model_path = "../data/processed/my_rf_model/DRF_model_python_1601519217841_129"
    
# load the model
my_rf = h2o.load_model(model_path)

# download the model built above to your local machine
# my_local_model = h2o.download_model(my_rf, path="../data/processed/my_rf_model")

# upload the model that you just downloded above
# to the H2O cluster
# uploaded_model = h2o.upload_model(my_local_model)
# Generate predictions on a test set (if neccessary)
pred = my_rf.predict(test)

pred
from h2o.estimators import H2ODeepLearningEstimator


# Build and train the model:
my_dl = H2ODeepLearningEstimator(distribution="multinomial",
                                 keep_cross_validation_predictions = True,
                                 nfolds= nfolds,
                                 fold_assignment = "random",
                                 seed=1234
)

my_dl.train(x= x, y= y, training_frame= train)
from h2o.estimators import H2OXGBoostEstimator

# NOTE make sure that the xgboost backend is setup
# NOTE xgboost is not supported on windows (only mac and linux)

# is_xgboost_available = H2OXGBoostEstimator.available()


my_xgb = H2OXGBoostEstimator(booster='dart',
                             normalize_type="tree",
                            keep_cross_validation_predictions = True,
                                 nfolds= nfolds,
                                 fold_assignment = "random",
                                seed=1234)

my_xgb.train(x=x, y=y, training_frame=train)

from h2o.estimators.stackedensemble import H2OStackedEnsembleEstimator


ensemble = H2OStackedEnsembleEstimator(
                                       base_models=[
                                                    my_gbm, 
                                                    my_rf, 
                                                    my_nb, 
                                                    my_dl,
#                                                     my_xgb,
                                       ])

ensemble.train(x=x, y=y, training_frame=train)


# Eval ensemble performance on the test data
perf_stack_test = ensemble.model_performance(test)

# Compare to base learner performance on the test set
perf_gbm_test = my_gbm.model_performance(test)
perf_rf_test = my_rf.model_performance(test)

baselearner_best_auc_test = max(perf_gbm_test.auc(), perf_rf_test.auc())
stack_auc_test = perf_stack_test.auc()

print("Best Base-learner Test AUC:  {0}".format(baselearner_best_auc_test))
print("Ensemble Test AUC:  {0}".format(stack_auc_test))


# Generate predictions on a test set (if neccessary)
pred = ensemble.predict(test)

pred
from h2o.grid.grid_search import H2OGridSearch

# 2. Generate a random grid of homogenous models and stack them together

# Specify GBM hyperparameters for the grid
hyper_params = {"learn_rate": [0.01, 0.03],
                "max_depth": [3, 4, 5, 6, 9],
                "sample_rate": [0.7, 0.8, 0.9, 1.0],
                "col_sample_rate": [0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8]}
search_criteria = {"strategy": "RandomDiscrete", "max_models": 3, "seed": 1}

# Train the grid
grid = H2OGridSearch(model=H2OGradientBoostingEstimator(ntrees=10,
                                                        seed=1234,
                                                        nfolds=nfolds,
                                                        fold_assignment="Modulo",
                                                        keep_cross_validation_predictions=True),
                     hyper_params=hyper_params,
                     search_criteria=search_criteria,
                     grid_id="gbm_grid_binomial")
grid.train(x=x, y=y, training_frame=train) 

# Train a stacked ensemble using the GBM grid
ensemble = H2OStackedEnsembleEstimator(model_id="my_ensemble_gbm_grid_binomial",
                                       base_models=grid.model_ids)
ensemble.train(x=x, y=y, training_frame=train)

from h2o.automl import H2OAutoML

# Run AutoML for 20 base models (limited to 1 hour max runtime by default)
aml = H2OAutoML(max_models=20, seed=1234, stopping_metric= 'AUTO')
aml.train(x=x, y=y, training_frame=train)

# View the AutoML Leaderboard
lb = aml.leaderboard
lb.head(rows=lb.nrows)  # Print all rows instead of default (10 rows) 


# Eval ensemble performance on the test data
perf_stack_test = ensemble.model_performance(test)

# Compare to base learner performance on the test set
baselearner_best_auc_test = max([h2o.get_model(m odel).model_performance(test_data=test).auc() for model in grid.model_ids])
stack_auc_test = perf_stack_test.auc()
print("Best Base-learner Test AUC:  {0}".format(baselearner_best_auc_test))
print("Ensemble Test AUC:  {0}".format(stack_auc_test))


# Generate predictions on a test set (if neccessary)
pred = ensemble.predict(test)

pred<jupyter_output>stackedensemble prediction progress: |████████████████████████████████████| 100%
{'fraction_non_alphanumeric': 0.0926039119804401, 'fraction_numerical': 0.0210880195599022, 'mean_word_length': 2.631890492045875, 'pattern_counts': {'":': 7, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 48, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '356172', 'license': 'MIT', 'n_tokens_mistral': 3310, 'n_tokens_neox': 3048, 'n_words': 686, 'path': 'src/notebooks/Model_Development.ipynb', 'repo_name': 'abhi18av/drug-resistance-prediction-hackathon', 'size': '9816'}