<commit_before>import unittest, random, sys, time, math
sys.path.extend(['.','..','py'])

import h2o, h2o_cmd, h2o_hosts, h2o_browse as h2b, h2o_import as h2i, h2o_glm


def gen_rand_equation(colCount, SEED):
    r1 = random.Random(SEED)
    coefficients = []
    # y = 1/(1 + exp(-(sum(coefficients*x)+intercept))
    for j in range(colCount):
        # ri = r1.randint(-1,1)
        rif = r1.uniform(-1,1)
        # rif = (j+0.0)/colCount # git bigger for each one
        coefficients.append(rif)
        # FIX! temporary try fixed = col+1
        # coefficients.append(j+1)
        # coefficients.append(2 + 2*(j%2))

    intercept = r1.uniform(-1,1)
    # intercept =  0
    print "Expected coefficients:", coefficients
    print "Expected intercept:", intercept

    return(coefficients, intercept) 

# FIX! random noise on coefficients? randomly force 5% to 0?  
#y = 1/(1 + math.exp(-(sum(coefficients*x)+intercept)) 
def yFromEqnAndData(coefficients, intercept, rowData):
    # FIX! think about using noise on some of the rowData
    cx = [a*b for a,b in zip(coefficients, rowData)]
    y = 1/(1 + math.exp(-(sum(cx) + intercept)))
    if (y<0 or y>1):
        raise Exception("Generated y result is should be between 0 and 1: " + y)
    return y

def write_syn_dataset(csvPathname, rowCount, colCount, coefficients, intercept, SEED, noise=0.05):
    r1 = random.Random(SEED)
    dsf = open(csvPathname, "w+")

    # assuming output is always last col
    yMin = None  
    yMax = None
    BINS = 100
    print "gen'ed y will be a probability! generate 1/0 data rows to reflect that probability, binned to %d bins" % BINS
    print "100 implies 2 places of accuracy in getting the probability." 
    print  "this means we should get 1 place of accuracy in the result coefficients/intercept????"
    # generate a mode per column that is reused
    # this will make every column have a different data distribution
    colModes = [r1.uniform(-1,1) for j in range(colCount)]
    colModes = [((random.randint(0,1) * -1) * j/colCount) for j in range(colCount)]
    for i in range(rowCount):
        rowData = []
        for j in range(colCount):
            # ri = r1.uniform(0,1) 
            ri = r1.triangular(-1,1,colModes[j])
            rowData.append(ri)
        
        # flip if within the noise percentage
        # FIX! not used right now
        if (noise is not None) and (r1.random() <= noise): 
            flip = True
        else: 
            flip = False
        
        # Do a walk from 0 to 1 by .1
        # writing 0 or 1 depending on whether you are below or above the probability
        # coarse approximation to get better coefficient match in GLM
        y = yFromEqnAndData(coefficients, intercept, rowData)

        if yMin is None or y<yMin: yMin = y
        if yMax is None or y>yMax: yMax = y
        
        for i in range(1,BINS+1): # 10 bins
            if y > (i + 0.0)/BINS:
                binomial = 1
            else:
                binomial = 0
            rowDataCsv = ",".join(map(str,rowData + [binomial]))
            dsf.write(rowDataCsv + "\n")

    dsf.close()
    print "yMin:", yMin, " yMax:", yMax


class Basic(unittest.TestCase):
    def tearDown(self):
        h2o.check_sandbox_for_errors()

    @classmethod
    def setUpClass(cls):
        global SEED, localhost
        SEED = h2o.setup_random_seed()
        localhost = h2o.decide_if_localhost()
        if (localhost):
            h2o.build_cloud(2,java_heap_GB=6)
        else:
            h2o_hosts.build_cloud_with_hosts()

    @classmethod
    def tearDownClass(cls):
        ### time.sleep(3600)
        h2o.tear_down_cloud()

    def test_GLM_with_logit_result_1(self):
        SYNDATASETS_DIR = h2o.make_syn_dir()
        tryList = [
            (100, 1, 'cA', 300), 
            # (100, 25, 'cB', 300), 
            # (1000, 25, 'cC', 300), <commit_msg>make it a 1 jvm test again<commit_after>import unittest, random, sys, time, math
sys.path.extend(['.','..','py'])

import h2o, h2o_cmd, h2o_hosts, h2o_browse as h2b, h2o_import as h2i, h2o_glm


def gen_rand_equation(colCount, SEED):
    r1 = random.Random(SEED)
    coefficients = []
    # y = 1/(1 + exp(-(sum(coefficients*x)+intercept))
    for j in range(colCount):
        # ri = r1.randint(-1,1)
        rif = r1.uniform(-1,1)
        # rif = (j+0.0)/colCount # git bigger for each one
        coefficients.append(rif)
        # FIX! temporary try fixed = col+1
        # coefficients.append(j+1)
        # coefficients.append(2 + 2*(j%2))

    intercept = r1.uniform(-1,1)
    # intercept =  0
    print "Expected coefficients:", coefficients
    print "Expected intercept:", intercept

    return(coefficients, intercept) 

# FIX! random noise on coefficients? randomly force 5% to 0?  
#y = 1/(1 + math.exp(-(sum(coefficients*x)+intercept)) 
def yFromEqnAndData(coefficients, intercept, rowData):
    # FIX! think about using noise on some of the rowData
    cx = [a*b for a,b in zip(coefficients, rowData)]
    y = 1/(1 + math.exp(-(sum(cx) + intercept)))
    if (y<0 or y>1):
        raise Exception("Generated y result is should be between 0 and 1: " + y)
    return y

def write_syn_dataset(csvPathname, rowCount, colCount, coefficients, intercept, SEED, noise=0.05):
    r1 = random.Random(SEED)
    dsf = open(csvPathname, "w+")

    # assuming output is always last col
    yMin = None  
    yMax = None
    BINS = 100
    print "gen'ed y will be a probability! generate 1/0 data rows to reflect that probability, binned to %d bins" % BINS
    print "100 implies 2 places of accuracy in getting the probability." 
    print  "this means we should get 1 place of accuracy in the result coefficients/intercept????"
    # generate a mode per column that is reused
    # this will make every column have a different data distribution
    colModes = [r1.uniform(-1,1) for j in range(colCount)]
    colModes = [((random.randint(0,1) * -1) * j/colCount) for j in range(colCount)]
    for i in range(rowCount):
        rowData = []
        for j in range(colCount):
            # ri = r1.uniform(0,1) 
            ri = r1.triangular(-1,1,colModes[j])
            rowData.append(ri)
        
        # flip if within the noise percentage
        # FIX! not used right now
        if (noise is not None) and (r1.random() <= noise): 
            flip = True
        else: 
            flip = False
        
        # Do a walk from 0 to 1 by .1
        # writing 0 or 1 depending on whether you are below or above the probability
        # coarse approximation to get better coefficient match in GLM
        y = yFromEqnAndData(coefficients, intercept, rowData)

        if yMin is None or y<yMin: yMin = y
        if yMax is None or y>yMax: yMax = y
        
        for i in range(1,BINS+1): # 10 bins
            if y > (i + 0.0)/BINS:
                binomial = 1
            else:
                binomial = 0
            rowDataCsv = ",".join(map(str,rowData + [binomial]))
            dsf.write(rowDataCsv + "\n")

    dsf.close()
    print "yMin:", yMin, " yMax:", yMax


class Basic(unittest.TestCase):
    def tearDown(self):
        h2o.check_sandbox_for_errors()

    @classmethod
    def setUpClass(cls):
        global SEED, localhost
        SEED = h2o.setup_random_seed()
        localhost = h2o.decide_if_localhost()
        if (localhost):
            h2o.build_cloud(1,java_heap_GB=12)
        else:
            h2o_hosts.build_cloud_with_hosts()

    @classmethod
    def tearDownClass(cls):
        ### time.sleep(3600)
        h2o.tear_down_cloud()

    def test_GLM_with_logit_result_1(self):
        SYNDATASETS_DIR = h2o.make_syn_dir()
        tryList = [
            (100, 1, 'cA', 300), 
            # (100, 25, 'cB', 300), 
            # (1000, 25, 'cC', 300), 
{'fraction_non_alphanumeric': 0.11319783895034731, 'fraction_numerical': 0.030100334448160536, 'mean_word_length': 2.3383426363246027, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5522327', 'n_tokens_mistral': 2723, 'n_tokens_neox': 2528, 'n_words': 768}