<commit_before>import unittest
import random, sys, time, re
sys.path.extend(['.','..','py'])

import h2o, h2o_cmd, h2o_hosts, h2o_browse as h2b, h2o_import2 as h2i, h2o_glm, h2o_util, h2o_rf
class Basic(unittest.TestCase):
    def tearDown(self):
        h2o.check_sandbox_for_errors()

    @classmethod
    def setUpClass(cls):
        h2o_hosts.build_cloud_with_hosts()

    @classmethod
    def tearDownClass(cls):
        h2o.tear_down_cloud()

    def test_GBM_parseTrain(self):
        #folderpath, filename, keyname, timeout
        bucket = 'home-0xdiag-datasets'
        files = [('mnist/mnist8m', 'mnist8m-train-1.csv', 'mnist8mtrain.hex', 1800, 0),
                 ('airlines', '1988_2008.csv', 'airlines.hex', 1800,'LateAircraftDelay'),
                 ('standard', 'covtype200x.data', 'covtype.hex', 1800, 55)
                ]
        for importFolderPath,csvFilename,trainKey,timeoutSecs,vresponse in files:
            # PARSE train****************************************
            start = time.time()
            parseResult = h2i.import_parse(bucket=bucket, path=importFolderPath + "/" + csvFilename,
                hex_key=trainKey, timeoutSecs=timeoutSecs)
            elapsed = time.time() - start
            print "parse end on ", csvFilename, 'took', elapsed, 'seconds',\
                "%d pct. of timeout" % ((elapsed*100)/timeoutSecs)
            print "parse result:", parseResult['destination_key']

            # GBM (train)****************************************
            params = { 
                'destination_key': "GBMKEY",
                'learn_rate':.1,
                'ntrees':10,
                'max_depth':8,
                'min_rows':1,
                'vresponse':vresponse
            }   

            kwargs = params.copy()
            h2o.beta_features = True
            #noPoll -> False when GBM finished<commit_msg>added more files for GBM to test against<commit_after>import unittest
import random, sys, time, re
sys.path.extend(['.','..','py'])

import h2o, h2o_cmd, h2o_hosts, h2o_browse as h2b, h2o_import2 as h2i, h2o_glm, h2o_util, h2o_rf
class Basic(unittest.TestCase):
    def tearDown(self):
        h2o.check_sandbox_for_errors()

    @classmethod
    def setUpClass(cls):
        h2o_hosts.build_cloud_with_hosts()

    @classmethod
    def tearDownClass(cls):
        h2o.tear_down_cloud()

    def test_GBM_parseTrain(self):
        #folderpath, filename, keyname, timeout
        bucket = 'home-0xdiag-datasets'
        
        files = [('airlines', '1988.csv', '1988.hex',1800,'LateAircraftDelay'),
                 ('mnist', 'mnist_training.csv.gz', 'mnistsmalltrain.hex',1800,784),
                 ('manyfiles-nflx-gz', 'file_95.dat.gz', 'nflx.hex',1800,541),
                 ('mnist/mnist8m', 'mnist8m-train-1.csv', 'mnist8mtrain.hex', 1800, 0),
                 ('airlines', '1988_2008.csv', 'airlines.hex', 1800,'LateAircraftDelay'),
                 ('standard', 'covtype200x.data', 'covtype.hex', 1800, 55)
                ]
        for importFolderPath,csvFilename,trainKey,timeoutSecs,vresponse in files:
            # PARSE train****************************************
            start = time.time()
            parseResult = h2i.import_parse(bucket=bucket, path=importFolderPath + "/" + csvFilename,
                hex_key=trainKey, timeoutSecs=timeoutSecs)
            elapsed = time.time() - start
            print "parse end on ", csvFilename, 'took', elapsed, 'seconds',\
                "%d pct. of timeout" % ((elapsed*100)/timeoutSecs)
            print "parse result:", parseResult['destination_key']

            # GBM (train)****************************************
            params = { 
                'destination_key': "GBMKEY",
                'learn_rate':.1,
                'ntrees':10,
                'max_depth':8,
                'min_rows':1,
                'vresponse':vresponse
            }   

            kwargs = params.copy()
            h2o.beta_features = True
            #noPoll -> False when GBM finished
{'fraction_non_alphanumeric': 0.16163366336633664, 'fraction_numerical': 0.03415841584158416, 'mean_word_length': 2.2773722627737225, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7473082', 'n_tokens_mistral': 1358, 'n_tokens_neox': 1229, 'n_words': 235}