<jupyter_start><jupyter_text>License ***Copyright (C) 2017 <NAME>, <EMAIL>Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. *** Decision Tree Ensemble Examples Kaggle House Prices   Imports and inits<jupyter_code>import h2o
from h2o.estimators.gbm import H2OGradientBoostingEstimator 
from h2o.estimators.random_forest import H2ORandomForestEstimator
from h2o.grid.grid_search import H2OGridSearch 
from h2o.estimators.xgboost import H2OXGBoostEstimator
from h2o.estimators.stackedensemble import H2OStackedEnsembleEstimator
import xgboost as xgb
h2o.init() # give h2o as much memory as possible
h2o.no_progress() # turn off h2o progress bars

import numpy as np
import pandas as pd<jupyter_output>Checking whether there is an H2O instance running at http://localhost:54321. connected.<jupyter_text>Import data<jupyter_code>train = h2o.import_file('../../03_regression/data/train.csv')
test = h2o.import_file('../../03_regression/data/test.csv')

# bug fix - from Keston
dummy_col = np.random.rand(test.shape[0])
test = test.cbind(h2o.H2OFrame(dummy_col))
cols = test.columns
cols[-1] = 'SalePrice'
test.columns = cols
print(train.shape)
print(test.shape)<jupyter_output>(1460, 81)
(1459, 81)<jupyter_text>Determine data types<jupyter_code>def get_type_lists(frame=train, rejects=['Id', 'SalePrice']):

    """Creates lists of numeric and categorical variables.
    
    :param frame: The frame from which to determine types.
    :param rejects: Variable names not to be included in returned lists.
    :return: Tuple of lists for numeric and categorical variables in the frame.
    
    """
    
    nums, cats = [], []
    for key, val in frame.types.items():
        if key not in rejects:
            if val == 'enum':
                cats.append(key)
            else: 
                nums.append(key)
                
    print('Numeric =', nums)                
    print()
    print('Categorical =', cats)
    
    return nums, cats
original_nums, cats = get_type_lists()<jupyter_output>Numeric = ['FullBath', 'TotalBsmtSF', 'KitchenAbvGr', 'BsmtFinSF2', 'TotRmsAbvGrd', 'MSSubClass', 'MasVnrArea', 'GrLivArea', 'BsmtFullBath', 'YearRemodAdd', 'BsmtFinSF1', 'Fireplaces', '2ndFlrSF', 'MoSold', 'ScreenPorch', 'PoolArea', 'OverallQual', '3SsnPorch', 'YrSold', 'LowQualFinSF', 'BsmtHalfBath', 'GarageYrBlt', 'EnclosedPorch', 'LotFrontage', 'WoodDeckSF', 'MiscVal', 'BsmtUnfSF', 'GarageArea', 'BedroomAbvGr', 'OpenPorchSF', 'LotArea', 'OverallCond', 'GarageCars', 'HalfBath', 'YearBuilt', '1stFlrSF']

Categorical = ['KitchenQual', 'Electrical', 'LotShape', 'SaleType', 'LandContour', 'LandSlope', 'GarageType', 'BsmtExposure', 'MasVnrType', 'GarageCond', 'ExterQual', 'Street', 'Functional', 'Alley', 'BsmtQual', 'HeatingQC', 'BsmtFinType2', 'RoofMatl', 'GarageQual', 'CentralAir', 'GarageFinish', 'HouseStyle', 'RoofStyle', 'Utilities', 'Neighborhood', 'Condition2', 'Exterior2nd', 'BldgType', 'SaleCondition', 'BsmtCond', 'LotConfig', 'Condition1', 'Fence', 'Foundation', 'PoolQC', 'Pave[...]<jupyter_text>Split into to train and validation (before doing data prep!!!)<jupyter_code>train, valid = train.split_frame([0.7], seed=12345)
print(train.shape)
print(valid.shape)<jupyter_output>(1001, 81)
(459, 81)<jupyter_text>Impute numeric missing<jupyter_code># median usually better than mean
# (_ signifies temporary throw-away variable, used to suppress output)
_ = train[['MasVnrArea', 'GarageYrBlt', 'LotFrontage']].impute(method='median')
_ = valid[['MasVnrArea', 'GarageYrBlt', 'LotFrontage']].impute(method='median')
_ = test[['BsmtHalfBath', 'BsmtFinSF1', 'BsmtFullBath', 'BsmtFinSF2', 'BsmtUnfSF', 'MasVnrArea', 
          'GarageYrBlt', 'LotFrontage', 'GarageCars', 'TotalBsmtSF', 'GarageArea']].impute(method='median')<jupyter_output><empty_output><jupyter_text>Encode categorical vars using shrunken averageshttp://helios.mm.di.uoa.gr/~rouvas/ssi/sigkdd/sigkdd.vol3.1/barreca.ps<jupyter_code>def target_encoder(training_frame, test_frame, x, y, lambda_=0.15, threshold=150, test=False):

    """ Applies simple target encoding to categorical variables.

    :param training_frame: Training frame which to create target means and to be encoded.
    :param test_frame: Test frame to be encoded using information from training frame.
    :param x: Name of input variable to be encoded.
    :param y: Name of target variable to use for encoding.
    :param lambda_: Balance between level mean and overall mean for small groups.
    :param threshold: Number below which a level is considered small enough to be shrunken.
    :param test: Whether or not to print the row_val_dict for testing purposes.
    :return: Tuple of encoded variable from train and test set as H2OFrames.

    """

    # convert to pandas
    trdf = training_frame.as_data_frame().loc[:, [x,y]] # df
    tss = test_frame.as_data_frame().loc[:, x]          # series


    # create dictionary of level:encode val

    encode_name = x + '_Tencode'
    overall_mean = trdf[y].mean()
    row_val_dict = {}

    for level in trdf[x].unique():
        level_df = trdf[trdf[x] == level][y]
        level_n = level_df.shape[0]
        level_mean = level_df.mean()
        if level_n >= threshold:
            row_val_dict[level] = level_mean
        else:
            row_val_dict[level] = ((1 - lambda_) * level_mean) +\
                                  (lambda_ * overall_mean)

    row_val_dict[np.nan] = overall_mean # handle missing values

    if test:
        print(row_val_dict)

    # apply the transform to training data
    trdf[encode_name] = trdf[x].apply(lambda i: row_val_dict[i])

    # apply the transform to test data
    tsdf = pd.DataFrame(columns=[x, encode_name])
    tsdf[x] = tss
    tsdf.loc[:, encode_name] = overall_mean # handle previously unseen values
    # handle values that are seen in tsdf but not row_val_dict
    for i, col_i in enumerate(tsdf[x]):
        try:
            row_val_dict[col_i]
        except:
            # a value that appeared in tsdf isn't in the row_val_dict so just
            # make it the overall_mean
            row_val_dict[col_i] = overall_mean
    tsdf[encode_name] = tsdf[x].apply(lambda i: row_val_dict[i])


    # convert back to H2O

    trdf = h2o.H2OFrame(trdf[encode_name].as_matrix())
    trdf.columns = [encode_name]

    tsdf = h2o.H2OFrame(tsdf[encode_name].as_matrix())
    tsdf.columns = [encode_name]

    return (trdf, tsdf)<jupyter_output><empty_output><jupyter_text>Execute encoding<jupyter_code>total = len(cats)
for i, var in enumerate(cats):
    
    tr_enc, _ = target_encoder(train, test, var, 'SalePrice')
    v_enc, ts_enc = target_encoder(valid, test, var, 'SalePrice')
    
    print('Encoding: ' + var + ' (' + str(i+1) + '/' + str(total) + ') ...')

    train = train.cbind(tr_enc)
    valid = valid.cbind(v_enc)
    test = test.cbind(ts_enc)    
    
print('Done.')<jupyter_output>Encoding: KitchenQual (1/43) ...
Encoding: Electrical (2/43) ...
Encoding: LotShape (3/43) ...
Encoding: SaleType (4/43) ...
Encoding: LandContour (5/43) ...
Encoding: LandSlope (6/43) ...
Encoding: GarageType (7/43) ...
Encoding: BsmtExposure (8/43) ...
Encoding: MasVnrType (9/43) ...
Encoding: GarageCond (10/43) ...
Encoding: ExterQual (11/43) ...
Encoding: Street (12/43) ...
Encoding: Functional (13/43) ...
Encoding: Alley (14/43) ...
Encoding: BsmtQual (15/43) ...
Encoding: HeatingQC (16/43) ...
Encoding: BsmtFinType2 (17/43) ...
Encoding: RoofMatl (18/43) ...
Encoding: GarageQual (19/43) ...
Encoding: CentralAir (20/43) ...
Encoding: GarageFinish (21/43) ...
Encoding: HouseStyle (22/43) ...
Encoding: RoofStyle (23/43) ...
Encoding: Utilities (24/43) ...
Encoding: Neighborhood (25/43) ...
Encoding: Condition2 (26/43) ...
Encoding: Exterior2nd (27/43) ...
Encoding: BldgType (28/43) ...
Encoding: SaleCondition (29/43) ...
Encoding: BsmtCond (30/43) ...
Encoding: LotConfig (31/43) ...[...]<jupyter_text>Redefine numerics and explore<jupyter_code>encoded_nums, cats = get_type_lists(frame=train)
print('Imputed and encoded numeric training data:')
train[encoded_nums].describe() #79 numeric columns w/ no missing
print('--------------------------------------------------------------------------------')
print('Imputed and encoded numeric validation data:')
valid[encoded_nums].describe() #79 numeric columns w/ no missing
print('--------------------------------------------------------------------------------')
print('Imputed and encoded numeric test data:')
test[encoded_nums].describe() #79 numeric columns w/ no missing
# Check Neighborhood_Tencode

print(test[0:5, ['Neighborhood', 'Neighborhood_Tencode']])
_, _ = target_encoder(valid, test, 'Neighborhood', 'SalePrice', test=True)
del _

# NAmes   152080
# NAmes   152080
# Gilbert 191808
# Gilbert 191808
# StoneBr 288359<jupyter_output><empty_output><jupyter_text>Create combination features<jupyter_code>def feature_combiner(training_frame, test_frame, nums):
    
    """ Combines numeric features using simple arithmatic operations.
    
    :param training_frame: Training frame from which to generate features and onto which generated 
                           feeatures will be cbound.
    :param test_frame: Test frame from which to generate features and onto which generated 
                       feeatures will be cbound.
    :param nums: List of original numeric features from which to generate combined features.
    
    """

    total = len(nums)
    
    # convert to pandas
    train_df = training_frame.as_data_frame()
    test_df = test_frame.as_data_frame()
    
    for i, col_i in enumerate(nums):
        
        print('Combining: ' + col_i + ' (' + str(i+1) + '/' + str(total) + ') ...')        
        
        for j, col_j in enumerate(nums):
            
            # don't repeat (i*j = j*i)
            if i < j:
                
                # convert to pandas
                col_i_train_df = train_df[col_i]
                col_j_train_df = train_df[col_j]
                col_i_test_df = test_df[col_i]
                col_j_test_df = test_df[col_j] 

                # multiply, convert back to h2o
                train_df[str(col_i + '|' + col_j)] = col_i_train_df.values*col_j_train_df.values
                test_df[str(col_i + '|' + col_j)] = col_i_test_df.values*col_j_test_df.values
                
    print('Done.')
    
    # convert back to h2o
    
    print('Converting to H2OFrame ...')
    
    training_frame = h2o.H2OFrame(train_df)
    training_frame.columns = list(train_df)
    test_frame = h2o.H2OFrame(test_df)
    test_frame.columns = list(test_df)
    
    print('Done.')
    print()
    
    # conserve memory 
    del train_df
    del test_df 
    
    return training_frame, test_frame

train, _ = feature_combiner(train, test, encoded_nums)
valid, test = feature_combiner(valid, test, encoded_nums)<jupyter_output>Combining: ExterCond_Tencode (1/79) ...
Combining: TotalBsmtSF (2/79) ...
Combining: Electrical_Tencode (3/79) ...
Combining: KitchenAbvGr (4/79) ...
Combining: LandContour_Tencode (5/79) ...
Combining: FireplaceQu_Tencode (6/79) ...
Combining: Alley_Tencode (7/79) ...
Combining: BsmtFinSF2 (8/79) ...
Combining: Foundation_Tencode (9/79) ...
Combining: TotRmsAbvGrd (10/79) ...
Combining: MSSubClass (11/79) ...
Combining: MasVnrArea (12/79) ...
Combining: Condition2_Tencode (13/79) ...
Combining: GrLivArea (14/79) ...
Combining: BsmtFullBath (15/79) ...
Combining: YearRemodAdd (16/79) ...
Combining: BsmtFinSF1 (17/79) ...
Combining: BsmtExposure_Tencode (18/79) ...
Combining: Fireplaces (19/79) ...
Combining: 2ndFlrSF (20/79) ...
Combining: MSZoning_Tencode (21/79) ...
Combining: RoofMatl_Tencode (22/79) ...
Combining: MoSold (23/79) ...
Combining: ScreenPorch (24/79) ...
Combining: PoolArea (25/79) ...
Combining: Exterior1st_Tencode (26/79) ...
Combining: 3SsnPorch (27/79) ...
Combinin[...]<jupyter_text>Redefine numerics and explore<jupyter_code>encoded_combined_nums, cats = get_type_lists(frame=train)
# check number of created variables is correct
# 1 id column, 1 target column, 79 original + encoded numeric columns, 43 original categorical variables
# sum(range(1, 79)) combined variables
print(train.shape == (1001, sum(range(1, 79), (79 + 43 + 1 + 1))))
print(test.shape == (1459, sum(range(1, 79), (79 + 43 + 1 + 1))))
# check multiplication for a random column
ridx = np.random.choice(sum(range(1, 79)))
combined_only = [name for name in encoded_combined_nums if name not in encoded_nums]
combined_check_vars = combined_only[ridx].split('|')
combined_check_vars.append(combined_only[ridx])

print(train[736, combined_check_vars])
print(test[637, combined_check_vars])

print(train[736, combined_check_vars[0]]*train[736, combined_check_vars[1]])
print(test[637, combined_check_vars[0]]*test[637, combined_check_vars[1]])<jupyter_output><empty_output><jupyter_text>Train models<jupyter_code>h2o.show_progress() # turn on progress bars
# Check log transform - looks good
%matplotlib inline
train['SalePrice'].log().as_data_frame().hist()

# Execute log transform
train['SalePrice'] = train['SalePrice'].log()
valid['SalePrice'] = valid['SalePrice'].log()
print(train[0:3, 'SalePrice'])<jupyter_output><empty_output><jupyter_text>Define ranked predictions plot function<jupyter_code>def ranked_preds_plot(y, valid, preds):
    
    """ Generates ranked prediction plot.
    
    :param y: Name of target variable.
    :param valid: Name of validation H2OFrame.
    :param preds: Column vector of predictions to plot.

    """
    
    # plot top frame values
    preds.columns = ['predict']
    yhat_frame = valid.cbind(preds)
    print(yhat_frame[0:10, [y, 'predict']])

    # plot sorted predictions
    yhat_frame_df = yhat_frame[[y, 'predict']].as_data_frame()
    yhat_frame_df.sort_values(by='predict', inplace=True)
    yhat_frame_df.reset_index(inplace=True, drop=True)
    _ = yhat_frame_df.plot(title='Ranked Predictions Plot')<jupyter_output><empty_output><jupyter_text>Function to generate submission file<jupyter_code>import re
import time

def gen_submission(preds, test=test):

    """ Generates submission file for Kaggle House Prices contest.
    
    :param preds: Column vector of predictions.
    :param test: Test data.
    
    """
    
    # create time stamp
    time_stamp = re.sub('[: ]', '_', time.asctime())

    # create predictions column
    sub = test['Id'].cbind(preds.exp())
    sub.columns = ['Id', 'SalePrice']
    
    # save file for submission
    sub_fname = '../data/submission_' + str(time_stamp) + '.csv'
    h2o.download_csv(sub, sub_fname)<jupyter_output><empty_output><jupyter_text>Simple prediction blending function<jupyter_code>import os

def pred_blender(dir_, files):
    
    """ Performs simple blending of prediction files. 
    
    :param dir_: Directory in which files to be read are stored.
    :param files: List of prediction files to be blended.
    
    """
    
    # read predictions in files list and cbind
    for i, file in enumerate(files):
        if i == 0:
            df = pd.read_csv(dir_ + os.sep + file).drop('SalePrice', axis=1)
        col = pd.read_csv(dir_ + os.sep + file).drop('Id', axis=1)
        col.columns = ['SalePrice' + str(i)]
        df = pd.concat([df, col], axis=1)
        
    # create mean prediction    
    df['mean'] = df.iloc[:, 1:].mean(axis=1)
    print(df.head())
        
    # create time stamp
    time_stamp = re.sub('[: ]', '_', time.asctime())        
        
    # write new submission file    
    df = df[['Id', 'mean']]
    df.columns = ['Id', 'SalePrice']
    
    # save file for submission
    sub_fname = '../data/submission_' + str(time_stamp) + '.csv'
    df.to_csv(sub_fname, index=False)<jupyter_output><empty_output><jupyter_text>Random forest model - typically not tuned as much as GBM<jupyter_code># initialize rf model
rf_model1 = H2ORandomForestEstimator(
    ntrees=10000,                    
    max_depth=10, 
    col_sample_rate_per_tree=0.1,
    sample_rate=0.8,
    stopping_rounds=50,
    score_each_iteration=True,
    nfolds=3,
    keep_cross_validation_predictions=True,
    seed=12345)           

# train rf model
rf_model1.train(
    x=encoded_combined_nums,
    y='SalePrice',
    training_frame=train,
    validation_frame=valid)

# print model information
print(rf_model1)

rf_preds1_val = rf_model1.predict(valid)
ranked_preds_plot('SalePrice', valid, rf_preds1_val) # valid RMSE not so hot ...
rf_preds1_test = rf_model1.predict(test)
gen_submission(rf_preds1_test) # 0.14574 public leaderboard<jupyter_output>drf Model Build progress: |███████████████████████████████████████████████| 100%
Model Details
=============
H2ORandomForestEstimator :  Distributed Random Forest
Model Key:  DRF_model_python_1497530715156_39


ModelMetricsRegression: drf
** Reported on train data. **

MSE: 0.017103284870273683
RMSE: 0.13077952771849913
MAE: 0.08983148719488154
RMSLE: 0.010147981199023763
Mean Residual Deviance: 0.017103284870273683

ModelMetricsRegression: drf
** Reported on validation data. **

MSE: 0.018264657787037664
RMSE: 0.13514680087607572
MAE: 0.09885812331003446
RMSLE: 0.010437127656733918
Mean Residual Deviance: 0.018264657787037664

ModelMetricsRegression: drf
** Reported on cross-validation data. **

MSE: 0.018240553858393096
RMSE: 0.13505759459724245
MAE: 0.08959435620967451
RMSLE: 0.010509925297604954
Mean Residual Deviance: 0.018240553858393096
Cross-Validation Metrics Summary:<jupyter_text>Extremely random trees model - typically not tuned as much as GBM<jupyter_code># initialize extra trees model
ert_model1 = H2ORandomForestEstimator(
    ntrees=10000,                    
    max_depth=10, 
    col_sample_rate_per_tree=0.1,
    sample_rate=0.8,
    stopping_rounds=50,
    score_each_iteration=True,
    nfolds=3,
    keep_cross_validation_predictions=True,
    seed=12345,
    histogram_type='random') # <- this is what makes it ERT instead of RF

# train ert model
ert_model1.train(
    x=encoded_combined_nums,
    y='SalePrice',
    training_frame=train,
    validation_frame=valid)

# print model information/create submission
print(ert_model1)
ert_preds1_val = ert_model1.predict(valid)
ranked_preds_plot('SalePrice', valid, ert_preds1_val) # valid RMSE not so hot ...
ert_preds1_test = ert_model1.predict(test)
gen_submission(ert_preds1_test) # 0.14855 public leaderboard<jupyter_output>drf Model Build progress: |███████████████████████████████████████████████| 100%
Model Details
=============
H2ORandomForestEstimator :  Distributed Random Forest
Model Key:  DRF_model_python_1497530715156_40


ModelMetricsRegression: drf
** Reported on train data. **

MSE: 0.017324102070941275
RMSE: 0.13162105481624614
MAE: 0.0892460685529443
RMSLE: 0.01021143122248222
Mean Residual Deviance: 0.017324102070941275

ModelMetricsRegression: drf
** Reported on validation data. **

MSE: 0.018656079782938404
RMSE: 0.1365872606905139
MAE: 0.10084357998026773
RMSLE: 0.010555786838637508
Mean Residual Deviance: 0.018656079782938404

ModelMetricsRegression: drf
** Reported on cross-validation data. **

MSE: 0.018342256462386228
RMSE: 0.13543358690659502
MAE: 0.08997786478212023
RMSLE: 0.010550036662112572
Mean Residual Deviance: 0.018342256462386228
Cross-Validation Metrics Summary:<jupyter_text>H2O GBM model<jupyter_code># initialize H2O GBM
h2o_gbm_model = H2OGradientBoostingEstimator(
    ntrees = 10000,
    learn_rate = 0.005,
    sample_rate = 0.1, 
    col_sample_rate = 0.8,
    max_depth = 5,
    nfolds = 3,
    keep_cross_validation_predictions=True,
    stopping_rounds = 10,
    seed = 12345)

# execute training
h2o_gbm_model.train(x=encoded_combined_nums,
                    y='SalePrice',
                    training_frame=train,
                    validation_frame=valid)

# print model information/create submission
print(h2o_gbm_model)
h2o_gbm_preds1_val = h2o_gbm_model.predict(valid)
ranked_preds_plot('SalePrice', valid, h2o_gbm_preds1_val) # better validation error
h2o_gbm_preds1_test = h2o_gbm_model.predict(test)
gen_submission(h2o_gbm_preds1_test) # 0.15062 public leaderboard<jupyter_output>gbm Model Build progress: |███████████████████████████████████████████████| 100%
Model Details
=============
H2OGradientBoostingEstimator :  Gradient Boosting Machine
Model Key:  GBM_model_python_1497530715156_41


ModelMetricsRegression: gbm
** Reported on train data. **

MSE: 0.006176434976609031
RMSE: 0.07859029823463601
MAE: 0.05301111489980966
RMSLE: 0.006099179628794482
Mean Residual Deviance: 0.006176434976609031

ModelMetricsRegression: gbm
** Reported on validation data. **

MSE: 0.017546665790746808
RMSE: 0.13246382823528394
MAE: 0.10119308547315771
RMSLE: 0.010243903760409427
Mean Residual Deviance: 0.017546665790746808

ModelMetricsRegression: gbm
** Reported on cross-validation data. **

MSE: 0.01645365655874634
RMSE: 0.12827180734185648
MAE: 0.08578849396442542
RMSLE: 0.009938680530110027
Mean Residual Deviance: 0.01645365655874634
Cross-Validation Metrics Summary:<jupyter_text>Train H2O XGBoost - very new!!<jupyter_code># initialize XGB GBM
h2o_xgb_model = H2OXGBoostEstimator(
    ntrees = 10000,
    learn_rate = 0.005,
    sample_rate = 0.1, 
    col_sample_rate = 0.8,
    max_depth = 5,
    nfolds = 3,
    keep_cross_validation_predictions=True,
    stopping_rounds = 10,
    seed = 12345)

# execute training 
h2o_xgb_model.train(x=encoded_combined_nums,
                    y='SalePrice',
                    training_frame=train,
                    validation_frame=valid)

# print model information/create submission
print(h2o_xgb_model)
h2o_xgb_preds1_val = h2o_xgb_model.predict(valid)
ranked_preds_plot('SalePrice', valid, h2o_xgb_preds1_val) 
h2o_xgb_preds1_test = h2o_xgb_model.predict(test)
gen_submission(h2o_xgb_preds1_test) # 0.16494 on public leaderboard<jupyter_output>xgboost Model Build progress: |███████████████████████████████████████████| 100%
Model Details
=============
H2OXGBoostEstimator :  XGBoost
Model Key:  XGBoost_model_python_1497530715156_42


ModelMetricsRegression: xgboost
** Reported on train data. **

MSE: 0.00840358868692754
RMSE: 0.09167108970077502
MAE: 0.05488036133788087
RMSLE: 0.007108057792476213
Mean Residual Deviance: 0.00840358868692754

ModelMetricsRegression: xgboost
** Reported on validation data. **

MSE: 0.020605834537669652
RMSE: 0.1435473250801618
MAE: 0.11459932348047726
RMSLE: 0.011121763109013057
Mean Residual Deviance: 0.020605834537669652

ModelMetricsRegression: xgboost
** Reported on cross-validation data. **

MSE: 0.017285036067784806
RMSE: 0.13147256773861538
MAE: 0.08792207052895834
RMSLE: 0.010168982730659084
Mean Residual Deviance: 0.017285036067784806
Cross-Validation Metrics Summary:<jupyter_text>Create blend<jupyter_code># create XGBoost blend
pred_blender('../data',
            ['submission_Thu_Jun_15_15_58_31_2017.csv',
             'submission_Thu_Jun_15_16_01_59_2017.csv',
             'submission_Thu_Jun_15_16_27_42_2017.csv',
             'submission_Thu_Jun_15_17_07_26_2017.csv'])
# 0.14705 on public leaderboard<jupyter_output>Id     SalePrice0     SalePrice1     SalePrice2     SalePrice3  \
0  1461  119081.074863  118567.449551  111874.076414  108450.225192   
1  1462  150015.290818  144812.782606  152345.926941  147465.853860   
2  1463  176503.892538  176940.325397  168826.171846  168720.909452   
3  1464  185223.022830  185907.910302  185119.575064  177289.259208   
4  1465  192687.508653  189991.207625  179623.201624  173343.132782   

            mean  
0  114493.206505  
1  148659.963557  
2  172747.824808  
3  183384.941851  
4  183911.262671<jupyter_text>Train H2O stacked ensemble<jupyter_code>stack = H2OStackedEnsembleEstimator(training_frame=train, 
                                    validation_frame=valid, 
                                    base_models=[rf_model1, ert_model1, 
                                                 h2o_gbm_model])

stack.train(x=encoded_combined_nums,
            y='SalePrice',
            training_frame=train,
            validation_frame=valid)

# print model information/create submission
print(stack)
stack_preds1_val = stack.predict(valid)
ranked_preds_plot('SalePrice', valid, stack_preds1_val) 
stack_preds1_test = stack.predict(test)
gen_submission(stack_preds1_test)
# 0.14630 on public leaderboard<jupyter_output>stackedensemble Model Build progress: |███████████████████████████████████| 100%
Model Details
=============
H2OStackedEnsembleEstimator :  Stacked Ensemble
Model Key:  StackedEnsemble_model_python_1497530715156_43
No model summary for this model


ModelMetricsRegressionGLM: stackedensemble
** Reported on train data. **

MSE: 0.0037195951897363083
RMSE: 0.06098848407475225
MAE: 0.04305787492087697
RMSLE: 0.004733690637851762
R^2: 0.9760134192704554
Mean Residual Deviance: 0.0037195951897363083
Null degrees of freedom: 1000
Residual degrees of freedom: 997
Null deviance: 155.22490791465145
Residual deviance: 3.7233147849260444
AIC: -2749.019633542589

ModelMetricsRegressionGLM: stackedensemble
** Reported on validation data. **

MSE: 0.016980254787969905
RMSE: 0.13030830667294355
MAE: 0.09925529488015572
RMSLE: 0.01009315309820752
R^2: 0.8992360780098589
Mean Residual Deviance: 0.016980254787969905
Null degrees of freedom: 458
Residual degrees of freedom: 455
Null deviance: 77.679978818[...]<jupyter_text>Native XGBoost GBM model (for example)<jupyter_code>"""

ave_y = half_train['SalePrice'].mean()[0]

# XGBoost uses SVMLight data structure, not Numpy arrays or Pandas data frames 
dtrain1 = xgb.DMatrix(train.as_data_frame()[encoded_combined_nums],
                      train.as_data_frame()['SalePrice'])
dvalid1 = xgb.DMatrix(valid.as_data_frame()[encoded_combined_nums],
                      valid.as_data_frame()['SalePrice'])
dtest = xgb.DMatrix(test.as_data_frame()[encoded_combined_nums])

# tuning parameters
params1 = {
    'objective': 'reg:linear',
    'booster': 'gbtree', 
    'eval_metric': 'rmse',
    'eta': 0.005,
    'subsample': 0.1, 
    'colsample_bytree': 0.8,
    'max_depth': 5,
    'reg_alpha' : 0.007,
    'reg_lambda' : 0.0,
    'base_score': ave_y,
    'silent': 0,
    'seed': 12345,
}

# watchlist is used for early stopping
watchlist = [(dtrain1, 'train'), (dvalid1, 'eval')]

# train model
xgb_model1 = xgb.train(params1, 
                       dtrain1, 
                       10000,
                       evals=watchlist, 
                       early_stopping_rounds=50, 
                       verbose_eval=True)

# create assessment stats and submission file
xgb_preds1_val = h2o.H2OFrame(xgb_model1.predict(dvalid1).tolist())
ranked_preds_plot('SalePrice', valid, xgb_preds1_val) 
xgb_preds1_test = h2o.H2OFrame(xgb_model1.predict(dtest).tolist())
gen_submission(xgb_preds1_test) 

"""<jupyter_output>[0]	train-rmse:0.394481	eval-rmse:0.386675
Multiple eval metrics have been passed: 'eval-rmse' will be used for early stopping.

Will train until eval-rmse hasn't improved in 50 rounds.
[1]	train-rmse:0.392907	eval-rmse:0.385508
[2]	train-rmse:0.391742	eval-rmse:0.384175
[3]	train-rmse:0.390294	eval-rmse:0.382704
[4]	train-rmse:0.388924	eval-rmse:0.381541
[5]	train-rmse:0.387633	eval-rmse:0.380214
[6]	train-rmse:0.386467	eval-rmse:0.379089
[7]	train-rmse:0.385057	eval-rmse:0.377412
[8]	train-rmse:0.383782	eval-rmse:0.376077
[9]	train-rmse:0.382354	eval-rmse:0.374844
[10]	train-rmse:0.381072	eval-rmse:0.373659
[11]	train-rmse:0.379447	eval-rmse:0.372006
[12]	train-rmse:0.377973	eval-rmse:0.370835
[13]	train-rmse:0.376468	eval-rmse:0.369565
[14]	train-rmse:0.374766	eval-rmse:0.368023
[15]	train-rmse:0.373482	eval-rmse:0.366748
[16]	train-rmse:0.372088	eval-rmse:0.365415
[17]	train-rmse:0.370921	eval-rmse:0.364812
[18]	train-rmse:0.369993	eval-rmse:0.364129
[19]	train-rmse:0.36835	eval-rm[...]<jupyter_text>Shutdown H2O<jupyter_code># Shutdown H2O - this will erase all your unsaved frames and models in H2O
h2o.cluster().shutdown(prompt=True)<jupyter_output>Are you sure you want to shutdown the H2O instance running at http://localhost:54321 (Y/N)? y
H2O session _sid_942e closed.
{'fraction_non_alphanumeric': 0.12456854894380781, 'fraction_numerical': 0.0987850338257628, 'mean_word_length': 3.9382989602863474, 'pattern_counts': {'":': 0, '<': 79, '<?xml version=': 0, '>': 78, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '22', 'dir': 'jupyter-structured-clean-dedup', 'id': '85586', 'license': 'Apache-2.0', 'n_tokens_mistral': 12334, 'n_tokens_neox': 10386, 'n_words': 2298, 'path': '04_decision_trees/src/py_part_4_kaggle_xgboost.ipynb', 'repo_name': 'rvsp/GWU_data_mining', 'size': '28988'}