<commit_before>import h2o_cmd, h2o
import re, random, math

def pickRandGlmParams(paramDict, params):
    colX = 0
    randomGroupSize = random.randint(1,len(paramDict))
    for i in range(randomGroupSize):
        randomKey = random.choice(paramDict.keys())
        randomV = paramDict[randomKey]
        randomValue = random.choice(randomV)
        params[randomKey] = randomValue
        if (randomKey=='x'):
            colX = randomValue

        # force legal family/ink combos
        if 'family' in params and 'link' in params: 
            if params['family'] is not None:
                if params['family'] == 'poisson':
                    if params['link'] is not None and params['link'] not in ('identity', 'log', 'inverse', 'familyDefault'):
                        params['link'] = None 
                # only tweedie/tweedie is legal?
                if params['family'] == 'tweedie':
                    if params['link'] is not None and params['link'] not in ('tweedie'):
                        params['link'] = None
                if params['family'] == 'binomial':
                    if params['link'] is not None and params['link'] not in ('logit', 'identity', 'log', 'inverse', 'familyDefault'):
                        params['link'] = None
                if params['family'] == 'gaussian':
                    if params['link'] is not None and params['link'] not in ('logit', 'identity', 'log', 'inverse', 'familyDefault'):
                        params['link'] = None

        # case only used if binomial? binomial is default if no family
        if 'family' not in params or params['family'] == 'binomial':
            maxCase = max(paramDict['case'])
            minCase = min(paramDict['case'])
            # make sure the combo of case and case_mode makes sense
            # there needs to be some entries in both effective cases
            if ('case_mode' in params):
                if ('case' not in params) or (params['case'] is None):
                    params['case'] = 1
                elif params['case_mode']=="<" and params['case']==minCase:
                    params['case'] += 1
                elif params['case_mode']==">" and params['case']==maxCase:
                    params['case'] -= 1
                elif params['case_mode']==">=" and params['case']==minCase:
                    params['case'] += 1
                elif params['case_mode']=="<=" and params['case']==maxCase:
                    params['case'] -= 1

    return colX


def simpleCheckGLMScore(self, glmScore, family='gaussian', allowFailWarning=False, **kwargs):
    warnings = None
    if 'warnings' in glmScore:
        warnings = glmScore['warnings']
        # stop on failed
        x = re.compile("failed", re.IGNORECASE)
        # don't stop if fail to converge
        c = re.compile("converge", re.IGNORECASE)
        for w in warnings:
            print "\nwarning:", w
            if re.search(x,w) and not allowFailWarning: 
                if re.search(c,w):
                    # ignore the fail to converge warning now
                    pass
                else: 
                    # stop on other 'fail' warnings (are there any? fail to solve?
                    raise Exception(w)

    validation = glmScore['validation']
    print "%15s %s" % ("err:\t", validation['err'])
    print "%15s %s" % ("nullDev:\t", validation['nullDev'])
    print "%15s %s" % ("resDev:\t", validation['resDev'])

    # threshold only there if binomial?
    # auc only for binomial
    if family=="binomial":
        print "%15s %s" % ("auc:\t", validation['auc'])
        print "%15s %s" % ("threshold:\t", validation['threshold'])

    if family=="poisson" or family=="gaussian":
        print "%15s %s" % ("aic:\t", validation['aic'])

    if math.isnan(validation['err']):
        emsg = "Why is this err = 'nan'?? %6s %s" % ("err:\t", validation['err'])
        raise Exception(emsg)

    if math.isnan(validation['resDev']):
        emsg = "Why is this resDev = 'nan'?? %6s %s" % ("resDev:\t", validation['resDev'])
        raise Exception(emsg)

    if math.isnan(validation['nullDev']):
        emsg = "Why is this nullDev = 'nan'?? %6s %s" % ("nullDev:\t", validation['nullDev'])
        raise Exception(emsg)

def simpleCheckGLM(self, glm, colX, allowFailWarning=False, allowZeroCoeff=False,
    prettyPrint=False, noPrint=False, maxExpectedIterations=None, doNormalized=False, **kwargs):
    # if we hit the max_iter, that means it probably didn't converge. should be 1-maxExpectedIter

    # h2o GLM will verboseprint the result and print errors. 
    # so don't have to do that
    # different when cross validation  is used? No trainingErrorDetails?
    GLMModel = glm['GLMModel']
    warnings = None
    if 'warnings' in GLMModel:
        warnings = GLMModel['warnings']
        # stop on failed
        x = re.compile("failed", re.IGNORECASE)
        # don't stop if fail to converge
        c = re.compile("converge", re.IGNORECASE)
        for w in warnings:
            print "\nwarning:", w
            if re.search(x,w) and not allowFailWarning: 
                if re.search(c,w):
                    # ignore the fail to converge warning now
                    pass
                else: 
                    # stop on other 'fail' warnings (are there any? fail to solve?
                    raise Exception(w)

    # for key, value in glm.iteritems(): print key
    # not in GLMGrid?

    # FIX! don't get GLMParams if it can't solve?
    GLMParams = GLMModel["GLMParams"]
    family = GLMParams["family"]

    iterations = GLMModel['iterations']
    print "GLMModel/iterations:", iterations

            # if we hit the max_iter, that means it probably didn't converge. should be 1-maxExpectedIter
    if maxExpectedIterations is not None and iterations  > maxExpectedIterations:
            raise Exception("Convergence issue? GLM did iterations: %d which is greater than expected: %d" % (iterations, maxExpectedIterations) )

    # pop the first validation from the list
    validationsList = GLMModel['validations']
    # don't want to modify validationsList in case someone else looks at it
    validations = validationsList[0]

    # xval. compare what we asked for and what we got.
    n_folds = kwargs.setdefault('n_folds', None)
    if not 'xval_models' in validations:
        if n_folds > 1:
            raise Exception("No cross validation models returned. Asked for "+n_folds)
    else:
        xval_models = validations['xval_models']
        if n_folds and n_folds > 1:
            if len(xval_models) != n_folds:
                raise Exception(len(xval_models)+" cross validation models returned. Asked for "+n_folds)
        else:
            # should be default 10?
            if len(xval_models) != 10:
                raise Exception(str(len(xval_models))+" cross validation models returned. Default should be 10")

    print "GLMModel/validations"
    print "%15s %s" % ("err:\t", validations['err'])
    print "%15s %s" % ("nullDev:\t", validations['nullDev'])
    print "%15s %s" % ("resDev:\t", validations['resDev'])

    # threshold only there if binomial?
    # auc only for binomial
    if family=="binomial":
        print "%15s %s" % ("auc:\t", validations['auc'])
        print "%15s %s" % ("threshold:\t", validations['threshold'])

    if family=="poisson" or family=="gaussian":
        print "%15s %s" % ("aic:\t", validations['aic'])

    if math.isnan(validations['err']):
        emsg = "Why is this err = 'nan'?? %6s %s" % ("err:\t", validations['err'])
        raise Exception(emsg)

    if math.isnan(validations['resDev']):
        emsg = "Why is this resDev = 'nan'?? %6s %s" % ("resDev:\t", validations['resDev'])
        raise Exception(emsg)

    if math.isnan(validations['nullDev']):
        emsg = "Why is this nullDev = 'nan'?? %6s %s" % ("nullDev:\t", validations['nullDev'])
        raise Exception(emsg)

    # get a copy, so we don't destroy the original when we pop the intercept
    if doNormalized:
        coefficients = GLMModel['normalized_coefficients'].copy()
    else:
        coefficients = GLMModel['coefficients'].copy()

    column_names = GLMModel['column_names']
    # get the intercept out of there into it's own dictionary
    intercept = coefficients.pop('Intercept', None)
    # have to skip the output col! get it from kwargs
    # better always be there!
    y = kwargs['y']

<commit_msg>make nullDev = nan legal<commit_after>import h2o_cmd, h2o
import re, random, math


def cleanseNanInf(value):
    # h2o can pass Infinity and NaN strings where there should be a number. 
    # convert it back to a proper +-Inf or Nan

    translate = {
        '-Infinity': -float('Inf'), 
        'Infinity': float('Inf'), 
        'NaN': float('NaN'), 
    }


    if str(value) in translate:
        value = translate[str(value)]

    return value
    
def pickRandGlmParams(paramDict, params):
    colX = 0
    randomGroupSize = random.randint(1,len(paramDict))
    for i in range(randomGroupSize):
        randomKey = random.choice(paramDict.keys())
        randomV = paramDict[randomKey]
        randomValue = random.choice(randomV)
        params[randomKey] = randomValue
        if (randomKey=='x'):
            colX = randomValue

        # force legal family/ink combos
        if 'family' in params and 'link' in params: 
            if params['family'] is not None:
                if params['family'] == 'poisson':
                    if params['link'] is not None and params['link'] not in ('identity', 'log', 'inverse', 'familyDefault'):
                        params['link'] = None 
                # only tweedie/tweedie is legal?
                if params['family'] == 'tweedie':
                    if params['link'] is not None and params['link'] not in ('tweedie'):
                        params['link'] = None
                if params['family'] == 'binomial':
                    if params['link'] is not None and params['link'] not in ('logit', 'identity', 'log', 'inverse', 'familyDefault'):
                        params['link'] = None
                if params['family'] == 'gaussian':
                    if params['link'] is not None and params['link'] not in ('logit', 'identity', 'log', 'inverse', 'familyDefault'):
                        params['link'] = None

        # case only used if binomial? binomial is default if no family
        if 'family' not in params or params['family'] == 'binomial':
            maxCase = max(paramDict['case'])
            minCase = min(paramDict['case'])
            # make sure the combo of case and case_mode makes sense
            # there needs to be some entries in both effective cases
            if ('case_mode' in params):
                if ('case' not in params) or (params['case'] is None):
                    params['case'] = 1
                elif params['case_mode']=="<" and params['case']==minCase:
                    params['case'] += 1
                elif params['case_mode']==">" and params['case']==maxCase:
                    params['case'] -= 1
                elif params['case_mode']==">=" and params['case']==minCase:
                    params['case'] += 1
                elif params['case_mode']=="<=" and params['case']==maxCase:
                    params['case'] -= 1

    return colX


def simpleCheckGLMScore(self, glmScore, family='gaussian', allowFailWarning=False, **kwargs):
    warnings = None
    if 'warnings' in glmScore:
        warnings = glmScore['warnings']
        # stop on failed
        x = re.compile("failed", re.IGNORECASE)
        # don't stop if fail to converge
        c = re.compile("converge", re.IGNORECASE)
        for w in warnings:
            print "\nwarning:", w
            if re.search(x,w) and not allowFailWarning: 
                if re.search(c,w):
                    # ignore the fail to converge warning now
                    pass
                else: 
                    # stop on other 'fail' warnings (are there any? fail to solve?
                    raise Exception(w)

    validation = glmScore['validation']

    # Just cleanse the ones we have trouble with? maybe not complete
    validations['err'] = cleanseNanInf(validations['err'])
    validations['nullDev'] = cleanseNanInf(validations['nullDev'])
    validations['resDev'] = cleanseNanInf(validations['resDev'])
    print "%15s %s" % ("err:\t", validation['err'])
    print "%15s %s" % ("nullDev:\t", validation['nullDev'])
    print "%15s %s" % ("resDev:\t", validation['resDev'])

    # threshold only there if binomial?
    # auc only for binomial
    if family=="binomial":
        print "%15s %s" % ("auc:\t", validation['auc'])
        print "%15s %s" % ("threshold:\t", validation['threshold'])

    if family=="poisson" or family=="gaussian":
        print "%15s %s" % ("aic:\t", validation['aic'])

    if math.isnan(validation['err']):
        emsg = "Why is this err = 'nan'?? %6s %s" % ("err:\t", validation['err'])
        raise Exception(emsg)

    if math.isnan(validation['resDev']):
        emsg = "Why is this resDev = 'nan'?? %6s %s" % ("resDev:\t", validation['resDev'])
        raise Exception(emsg)

    # legal?
    if math.isnan(validation['nullDev']):
        ## emsg = "Why is this nullDev = 'nan'?? %6s %s" % ("nullDev:\t", validation['nullDev'])
        ## raise Exception(emsg)
        pass

def simpleCheckGLM(self, glm, colX, allowFailWarning=False, allowZeroCoeff=False,
    prettyPrint=False, noPrint=False, maxExpectedIterations=None, doNormalized=False, **kwargs):
    # if we hit the max_iter, that means it probably didn't converge. should be 1-maxExpectedIter

    # h2o GLM will verboseprint the result and print errors. 
    # so don't have to do that
    # different when cross validation  is used? No trainingErrorDetails?
    GLMModel = glm['GLMModel']
    warnings = None
    if 'warnings' in GLMModel:
        warnings = GLMModel['warnings']
        # stop on failed
        x = re.compile("failed", re.IGNORECASE)
        # don't stop if fail to converge
        c = re.compile("converge", re.IGNORECASE)
        for w in warnings:
            print "\nwarning:", w
            if re.search(x,w) and not allowFailWarning: 
                if re.search(c,w):
                    # ignore the fail to converge warning now
                    pass
                else: 
                    # stop on other 'fail' warnings (are there any? fail to solve?
                    raise Exception(w)

    # for key, value in glm.iteritems(): print key
    # not in GLMGrid?

    # FIX! don't get GLMParams if it can't solve?
    GLMParams = GLMModel["GLMParams"]
    family = GLMParams["family"]

    iterations = GLMModel['iterations']
    print "GLMModel/iterations:", iterations

            # if we hit the max_iter, that means it probably didn't converge. should be 1-maxExpectedIter
    if maxExpectedIterations is not None and iterations  > maxExpectedIterations:
            raise Exception("Convergence issue? GLM did iterations: %d which is greater than expected: %d" % (iterations, maxExpectedIterations) )

    # pop the first validation from the list
    validationsList = GLMModel['validations']
    # don't want to modify validationsList in case someone else looks at it
    validations = validationsList[0]

    # xval. compare what we asked for and what we got.
    n_folds = kwargs.setdefault('n_folds', None)
    if not 'xval_models' in validations:
        if n_folds > 1:
            raise Exception("No cross validation models returned. Asked for "+n_folds)
    else:
        xval_models = validations['xval_models']
        if n_folds and n_folds > 1:
            if len(xval_models) != n_folds:
                raise Exception(len(xval_models)+" cross validation models returned. Asked for "+n_folds)
        else:
            # should be default 10?
            if len(xval_models) != 10:
                raise Exception(str(len(xval_models))+" cross validation models returned. Default should be 10")

    print "GLMModel/validations"
    # Just cleanse the ones we have trouble with? maybe not complete
    validations['err'] = cleanseNanInf(validations['err'])
    validations['nullDev'] = cleanseNanInf(validations['nullDev'])
    validations['resDev'] = cleanseNanInf(validations['resDev'])
    print "%15s %s" % ("err:\t", validations['err'])
    print "%15s %s" % ("nullDev:\t", validations['nullDev'])
    print "%15s %s" % ("resDev:\t", validations['resDev'])

    # threshold only there if binomial?
    # auc only for binomial
    if family=="binomial":
        print "%15s %s" % ("auc:\t", validations['auc'])
        print "%15s %s" % ("threshold:\t", validations['threshold'])

    if family=="poisson" or family=="gaussian":
        print "%15s %s" % ("aic:\t", validations['aic'])

    if math.isnan(validations['err']):
        emsg = "Why is this err = 'nan'?? %6s %s" % ("err:\t", validations['err'])
        raise Exception(emsg)

    if math.isnan(validations['resDev']):
        emsg = "Why is this resDev = 'nan'?? %6s %s" % ("resDev:\t", validations['resDev'])
        raise Exception(emsg)

    # legal?
    if math.isnan(validations['nullDev']):
        ## emsg = "Why is this nullDev = 'nan'?? %6s %s" % ("nullDev:\t", validations['nullDev'])
        ## raise Exception(emsg)
        pass

    # get a copy, so we don't destroy the original when we pop the intercept
    if doNormalized:
        coefficients = GLMModel['normalized_coefficients'].copy()
    else:
        coefficients = GLMModel['coefficients'].copy()

    column_names = GLMModel['column_names']
    # get the intercept out of there into it's own dictionary
    intercept = coefficients.pop('Intercept', None)
    # have to skip the output col! get it from kwargs
    # better always be there!
    y = kwargs['y']


{'fraction_non_alphanumeric': 0.13014429285273146, 'fraction_numerical': 0.00578294312503509, 'mean_word_length': 2.4286814244465833, 'pattern_counts': {'":': 8, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7505834', 'n_tokens_mistral': 5368, 'n_tokens_neox': 5053, 'n_words': 1610}