public static void crossValidate ( Job . ValidatedJob job ) { if ( job . state != Job . JobState . RUNNING ) return ; //don't do cross-validation if the full model builder failed if ( job . validation != null ) throw new IllegalArgumentException ( "Cannot provide validation dataset and n_folds > 0 at the same time." ) ; if ( job . n_folds <= 1 ) throw new IllegalArgumentException ( "n_folds must be >= 2 for cross-validation." ) ; final String basename = job . destination_key . toString ( ) ; long [ ] offsets = new long [ job . n_folds + 1 ] ; Frame [ ] cv_preds = new Frame [ job . n_folds ] ; try { for ( int i = 0 ; i < job . n_folds ; ++ i ) { if ( job . state != Job . JobState . RUNNING ) break ; Key [ ] destkeys = new Key [ ] { Key . make ( basename + "_xval" + i + "_train" ) , Key . make ( basename + "_xval" + i + "_holdout" ) } ; NFoldFrameExtractor nffe = new NFoldFrameExtractor ( job . source , job . n_folds , i , destkeys , Key . make ( ) /*key used for locking only*/ ) ; H2O . submitTask ( nffe ) ; Frame [ ] splits = nffe . getResult ( ) ; // Cross-validate individual splits try { job . crossValidate ( splits , cv_preds , offsets , i ) ; //this removes the enum-ified response! job . _cv_count ++ ; } finally { // clean-up the results if ( ! job . keep_cross_validation_splits ) for ( Frame f : splits ) f . delete ( ) ; } } if ( job . state != Job . JobState . RUNNING ) return ; final int resp_idx = job . source . find ( job . _responseName ) ; Vec response = job . source . vecs ( ) [ resp_idx ] ; boolean put_back = UKV . get ( job . response . _key ) == null ; // In the case of rebalance, rebalance response will be deleted if ( put_back ) { job . response = response ; if ( job . classification ) job . response = job . response . toEnum ( ) ; DKV . put ( job . response . _key , job . response ) ; //put enum-ified response back to K-V store } ( ( Model ) UKV . get ( job . destination_key ) ) . scoreCrossValidation ( job , job . source , response , cv_preds , offsets ) ; if ( put_back ) UKV . remove ( job . response . _key ) ; } finally { // clean-up prediction frames for splits for ( Frame f : cv_preds ) if ( f != null ) f . delete ( ) ; } }
Cross - Validate a ValidatedJob