<commit_before>import unittest
import re, os, shutil, sys, random
sys.path.extend(['.','..','py'])

import h2o, h2o_cmd, h2o_hosts

def writeRows(csvPathname,row,eol,repeat):
    f = open(csvPathname, 'w')
    for r in range(repeat):
        f.write(row + eol)

class Basic(unittest.TestCase):
    def tearDown(self):
        h2o.check_sandbox_for_errors()

    @classmethod
    def setUpClass(cls):
        localhost = h2o.decide_if_localhost()
        if (localhost):
            h2o.build_cloud(2)
        else:
            h2o_hosts.build_cloud_with_hosts()

    @classmethod 
    def tearDownClass(cls): 
        h2o.tear_down_cloud()

    def test_A_parse_small_many(self):
        SEED = 6204672511291494176
        random.seed(SEED)
        print "\nUsing random seed:", SEED

        SYNDATASETS_DIR = h2o.make_syn_dir()
        # can try the other two possibilities also
        eol = "\n"
        row = "a,b,c,d,e,f,g"

        # need unique key name for upload and for parse, each time
        # maybe just upload it once?
        timeoutSecs = 10
        node = h2o.nodes[0]

        # fail rate is one in 200?
        # need at least two rows (parser)
        for sizeTrial in range(25):
            size = random.randint(2,129)
            print "\nparsing with rows:", size
            csvFilename = "p" + "_" + str(size)
            csvPathname = SYNDATASETS_DIR + "/" + csvFilename
            writeRows(csvPathname,row,eol,size)
            key  = csvFilename
            pkey = node.put_file(csvPathname, key=key, timeoutSecs=timeoutSecs)
            print h2o.dump_json(pkey)
            
            trialMax = 100
            for trial in range(trialMax):
                key2 = csvFilename + "_" + str(trial) + ".hex"
                # just parse, without polling, except for last one..will that make prior ones complete too?
                noPoll = trial==(trialMax-1)
                node.parse(pkey, key2, timeoutSecs=timeoutSecs, retryDelaySecs=0.00, noPoll=noPoll)
                if not trial%10:
                    sys.stdout.write('.')
                    sys.stdout.flush()

            # do a storeview ..was causing npe while parsing?
            # maybe fire to each node?
            if 1==1:
                for node in h2o.nodes:
                    storeView = node.store_view()



if __name__ == '__main__':
    h2o.unit_main()<commit_msg>longer datasets for parse<commit_after>import unittest
import re, os, shutil, sys, random
sys.path.extend(['.','..','py'])

import h2o, h2o_cmd, h2o_hosts

def writeRows(csvPathname,row,eol,repeat):
    f = open(csvPathname, 'w')
    for r in range(repeat):
        f.write(row + eol)

class Basic(unittest.TestCase):
    def tearDown(self):
        h2o.check_sandbox_for_errors()

    @classmethod
    def setUpClass(cls):
        localhost = h2o.decide_if_localhost()
        if (localhost):
            h2o.build_cloud(2)
        else:
            h2o_hosts.build_cloud_with_hosts()

    @classmethod 
    def tearDownClass(cls): 
        h2o.tear_down_cloud()

    def test_A_parse_small_many(self):
        SEED = 6204672511291494176
        random.seed(SEED)
        print "\nUsing random seed:", SEED

        SYNDATASETS_DIR = h2o.make_syn_dir()
        # can try the other two possibilities also
        eol = "\n"
        row = "a,b,c,d,e,f,g"

        # need unique key name for upload and for parse, each time
        # maybe just upload it once?
        timeoutSecs = 10
        node = h2o.nodes[0]

        # fail rate is one in 200?
        # need at least two rows (parser)
        for sizeTrial in range(25):
            size = random.randint(1129,2255)
            print "\nparsing with rows:", size
            csvFilename = "p" + "_" + str(size)
            csvPathname = SYNDATASETS_DIR + "/" + csvFilename
            writeRows(csvPathname,row,eol,size)
            key  = csvFilename
            pkey = node.put_file(csvPathname, key=key, timeoutSecs=timeoutSecs)
            print h2o.dump_json(pkey)
            
            trialMax = 100
            for trial in range(trialMax):
                key2 = csvFilename + "_" + str(trial) + ".hex"
                # just parse, without polling, except for last one..will that make prior ones complete too?
                noPoll = trial==(trialMax-1)
                node.parse(pkey, key2, timeoutSecs=timeoutSecs, retryDelaySecs=0.00, noPoll=noPoll)
                if not trial%10:
                    sys.stdout.write('.')
                    sys.stdout.flush()


            # a = h2o.nodes[0].jobs_admin()
            # print "jobs-admin():", h2o.dump_json(a)

            # do a storeview ..was causing npe while parsing?
            # maybe fire to each node?
            if 1==1:
                for node in h2o.nodes:
                    storeView = node.store_view()



if __name__ == '__main__':
    h2o.unit_main()
{'fraction_non_alphanumeric': 0.10348360655737705, 'fraction_numerical': 0.025204918032786884, 'mean_word_length': 1.9019024970273484, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4145380', 'n_tokens_mistral': 1660, 'n_tokens_neox': 1533, 'n_words': 377}