<commit_before>import time, sys, json, re, getpass, os, shutil, requests, datetime

import h2o_args
import h2o_print as h2p
from h2o_test import get_sandbox_name, check_sandbox_for_errors, dump_json, verboseprint

#*******************************************************************************
# duplicate do_json_request here, because the normal one is a method on a h2o node object which 
# doesn't exist yet. copied this from find_cloud.py
def create_url(addr, port, loc):
    return 'http://%s:%s/%s' % (addr, port, loc)

#*******************************************************************************
def do_json_request(addr=None, port=None,  jsonRequest=None, params=None, timeout=7, **kwargs):
    if params is not None:
        paramsStr =  '?' + '&'.join(['%s=%s' % (k,v) for (k,v) in params.items()])
    else:
        paramsStr = ''

    url = create_url(addr, port, jsonRequest)
    print 'Start ' + url + paramsStr
    try:
        r = requests.get(url, timeout=timeout, params=params, **kwargs)
        # the requests json decoder might fail if we didn't get something good
        rjson = r.json()
        emsg = "ERROR: Probing claimed existing cloud with Cloud.json"
        if not isinstance(rjson, (list,dict)):
            # probably good
            raise Exception(emsg + "h2o json responses should always be lists or dicts. Got %s" %\
                dump_json(rj))
        elif r.status_code != requests.codes.ok:
            rjson = None
            raise Exception(emsg + "Couldn't decode. Status: %s" % r.status_code)

    except requests.ConnectionError, e:
        rjson = None
        emsg = "ERROR: json got ConnectionError or other exception"
        # Rethrow the exception after we've checked for stack trace from h2o.
        # Out of memory errors maybe don't show up right away? 
        # so we should wait for h2o to get it out to h2o stdout. 
        # Don't want to rely on cloud teardown to check because there's no delay, 
        # and we don't want to delay all cloud teardowns by waiting.
        exc_info = sys.exc_info()
        # we don't expect to have connection errors, so any exception is a bad thing.
        h2p.red_print(
            "%s\n %s\n %s\nGoing to check sandbox, then rethrow.." % (emsg, exc_info, url + paramsStr))
        time.sleep(2)
        check_sandbox_for_errors()
        raise exc_info[1], None, exc_info[2]

    # print rjson
    return rjson

#*********************************************************************************************
def create_node(possMember, h2oNodes, expectedSize, hdfsSetup):

    (hdfs_version, hdfs_config, hdfs_name_node) = hdfsSetup

    http_addr, sep, port = possMember.rstrip('\n').partition(":")
    http_addr = http_addr.lstrip('/') # just in case it's an old-school flatfile format with leading /
    if port == '':
        port = '54321'
    if http_addr == '':
        http_addr = '127.0.0.1'
    # print "http_addr:", http_addr, "port:", port

    possMemberList = []
    gc = do_json_request(http_addr, port, '/1/Cloud.json', timeout=10)
    if gc is None:
        return possMemberList
        
    version    = gc['version']
    # check to see if it's a h2o-dev version? (common problem when mixing h2o1/h2o-dev testing with --usecloud
# local builds have (unknown)    if not version.startswith('0'):
# local builds have (unknown)        raise Exception("h2o version at node[0] doesn't look like h2o-dev version. (start with 0) %s" % version)

    # we'll just exception out, if we don't get a json response with the stuff that makes up what we think is "legal"
    consensus  = gc['consensus']
    locked     = gc['locked']
    cloud_size = gc['cloud_size']
    cloud_name = gc['cloud_name']
    nodes      = gc['nodes']

    # None means don't check
    if expectedSize and (cloud_size!=expectedSize):
        raise Exception("cloud_size %s at %s disagrees with -expectedSize %s" % \
            (cloud_size, cloud_name, expectedSize))

    print "here's some info about the java heaps in the cloud you said you already built for me"
    print "Also some info about ncpus"
    # will use these for comparing. All should be equal for normal clouds
    java_heap_GB_list = []
    num_cpus_list = []
    name_list = []
    for i, n in enumerate(nodes):
        print "max_mem (GB):", "%0.2f" % ((n['max_mem']+0.0)/(1024*1024*1024))
        print "tot_mem (GB):", "%0.2f" % ((n['tot_mem']+0.0)/(1024*1024*1024))
        java_heap_GB = (n['tot_mem']+0.0)/(1024*1024*1024)
        java_heap_GB = int(round(java_heap_GB,0))
        num_cpus = n['num_cpus']
        print "java_heap_GB:", java_heap_GB
        print 'num_cpus:', num_cpus

        java_heap_GB_list.append(java_heap_GB)
        num_cpus_list.append(num_cpus)

        name = n['h2o']['node'].lstrip('/')
        # print 'name:', name
        ### print dump_json(n)
        name_list.append(name)
        possMemberList.append(name)

        ip, sep, port = name.partition(':')

        # print "ip:", ip
        # print "port:", port
        if not ip or not port:
            raise Exception("bad ip or port parsing from h2o get_cloud nodes 'name' %s" % n['name'])

        # if you have more than one node, compare to the last one
        if i>0:
            lasti = i-1
            if java_heap_GB != java_heap_GB_list[lasti]:
                raise Exception("You have two nodes %s %s with different java heap sizes %s %s. \
                    Assuming that's bad/not your intent)" % \
                    (name[i], name[lasti], java_heap_GB, java_heap_GB_list[lasti]))
<commit_msg>Fix some py2 test mismatch with new json<commit_after>import time, sys, json, re, getpass, os, shutil, requests, datetime

import h2o_args
import h2o_print as h2p
from h2o_test import get_sandbox_name, check_sandbox_for_errors, dump_json, verboseprint

#*******************************************************************************
# duplicate do_json_request here, because the normal one is a method on a h2o node object which 
# doesn't exist yet. copied this from find_cloud.py
def create_url(addr, port, loc):
    return 'http://%s:%s/%s' % (addr, port, loc)

#*******************************************************************************
def do_json_request(addr=None, port=None,  jsonRequest=None, params=None, timeout=7, **kwargs):
    if params is not None:
        paramsStr =  '?' + '&'.join(['%s=%s' % (k,v) for (k,v) in params.items()])
    else:
        paramsStr = ''

    url = create_url(addr, port, jsonRequest)
    print 'Start ' + url + paramsStr
    try:
        r = requests.get(url, timeout=timeout, params=params, **kwargs)
        # the requests json decoder might fail if we didn't get something good
        rjson = r.json()
        emsg = "ERROR: Probing claimed existing cloud with Cloud.json"
        if not isinstance(rjson, (list,dict)):
            # probably good
            raise Exception(emsg + "h2o json responses should always be lists or dicts. Got %s" %\
                dump_json(rj))
        elif r.status_code != requests.codes.ok:
            rjson = None
            raise Exception(emsg + "Couldn't decode. Status: %s" % r.status_code)

    except requests.ConnectionError, e:
        rjson = None
        emsg = "ERROR: json got ConnectionError or other exception"
        # Rethrow the exception after we've checked for stack trace from h2o.
        # Out of memory errors maybe don't show up right away? 
        # so we should wait for h2o to get it out to h2o stdout. 
        # Don't want to rely on cloud teardown to check because there's no delay, 
        # and we don't want to delay all cloud teardowns by waiting.
        exc_info = sys.exc_info()
        # we don't expect to have connection errors, so any exception is a bad thing.
        h2p.red_print(
            "%s\n %s\n %s\nGoing to check sandbox, then rethrow.." % (emsg, exc_info, url + paramsStr))
        time.sleep(2)
        check_sandbox_for_errors()
        raise exc_info[1], None, exc_info[2]

    # print rjson
    return rjson

#*********************************************************************************************
def create_node(possMember, h2oNodes, expectedSize, hdfsSetup):

    (hdfs_version, hdfs_config, hdfs_name_node) = hdfsSetup

    http_addr, sep, port = possMember.rstrip('\n').partition(":")
    http_addr = http_addr.lstrip('/') # just in case it's an old-school flatfile format with leading /
    if port == '':
        port = '54321'
    if http_addr == '':
        http_addr = '127.0.0.1'
    # print "http_addr:", http_addr, "port:", port

    possMemberList = []
    gc = do_json_request(http_addr, port, '1/Cloud.json', timeout=10)
    if gc is None:
        return possMemberList
        
    version    = gc['version']
    # check to see if it's a h2o-dev version? (common problem when mixing h2o1/h2o-dev testing with --usecloud
# local builds have (unknown)    if not version.startswith('0'):
# local builds have (unknown)        raise Exception("h2o version at node[0] doesn't look like h2o-dev version. (start with 0) %s" % version)

    # we'll just exception out, if we don't get a json response with the stuff that makes up what we think is "legal"
    consensus  = gc['consensus']
    locked     = gc['locked']
    cloud_size = gc['cloud_size']
    cloud_name = gc['cloud_name']
    nodes      = gc['nodes']

    # None means don't check
    if expectedSize and (cloud_size!=expectedSize):
        raise Exception("cloud_size %s at %s disagrees with -expectedSize %s" % \
            (cloud_size, cloud_name, expectedSize))

    print "here's some info about the java heaps in the cloud you said you already built for me"
    print "Also some info about ncpus"
    # will use these for comparing. All should be equal for normal clouds
    java_heap_GB_list = []
    num_cpus_list = []
    name_list = []
    for i, n in enumerate(nodes):
        print "max_mem (GB):", "%0.2f" % ((n['max_mem']+0.0)/(1024*1024*1024))
        print "tot_mem (GB):", "%0.2f" % ((n['tot_mem']+0.0)/(1024*1024*1024))
        java_heap_GB = (n['tot_mem']+0.0)/(1024*1024*1024)
        java_heap_GB = int(round(java_heap_GB,0))
        num_cpus = n['num_cpus']
        print "java_heap_GB:", java_heap_GB
        print 'num_cpus:', num_cpus

        java_heap_GB_list.append(java_heap_GB)
        num_cpus_list.append(num_cpus)

        name = n['h2o'].lstrip('/')
        # print 'name:', name
        ### print dump_json(n)
        name_list.append(name)
        possMemberList.append(name)

        ip, sep, port = name.partition(':')

        # print "ip:", ip
        # print "port:", port
        if not ip or not port:
            raise Exception("bad ip or port parsing from h2o get_cloud nodes 'name' %s" % n['name'])

        # if you have more than one node, compare to the last one
        if i>0:
            lasti = i-1
            if java_heap_GB != java_heap_GB_list[lasti]:
                raise Exception("You have two nodes %s %s with different java heap sizes %s %s. \
                    Assuming that's bad/not your intent)" % \
                    (name[i], name[lasti], java_heap_GB, java_heap_GB_list[lasti]))

{'fraction_non_alphanumeric': 0.1533703836500538, 'fraction_numerical': 0.016045177482968805, 'mean_word_length': 2.8353386043313855, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1401062', 'n_tokens_mistral': 3602, 'n_tokens_neox': 3408, 'n_words': 1103}