<commit_before># tensoflow.__init__ calls _os.path.basename(_sys.argv[0])
# so we need to create a synthetic argv.
import sys
if not hasattr(sys, 'argv'):
    sys.argv  = ['p3b1']

import json
import os
import p3b1
import runner_utils
import socket
import time

node_pid = "%s,%i" % (socket.gethostname(), os.getpid())
print("node,pid: " + node_pid)

logger = None

def get_logger():
    """ Set up logging """
    global logger
    if logger is not None:
        return logger
    import logging, sys
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.DEBUG)
    h = logging.StreamHandler(stream=sys.stdout)
    fmtr = logging.Formatter('%(asctime)s %(name)s %(levelname)-9s %(message)s',
                             datefmt='%Y/%m/%d %H:%M:%S')
    h.setFormatter(fmtr)
    logger.addHandler(h)
    return logger

def run(hyper_parameter_map):

    logger = get_logger()
    framework = hyper_parameter_map['framework']
    logger.debug("IMPORT START " + str(time.time()))
    if framework == 'keras':
        import p3b1_baseline_keras2
        pkg = p3b1_baseline_keras2
    else:
        raise ValueError("Unsupported framework: {}".format(framework))
    logger.debug("IMPORT STOP")

    # params is python dictionary
    params = pkg.initialize_parameters()
    runner_utils.format_params(hyper_parameter_map)

    for k,v in hyper_parameter_map.items():
        #if not k in params:
        #    raise Exception("Parameter '{}' not found in set of valid arguments".format(k))
        params[k] = v

    logger.debug("WRITE_PARAMS START")
    runner_utils.write_params(params, hyper_parameter_map)
    logger.debug("WRITE_PARAMS STOP")
    logger.debug("DO_N_FOLD START " + str(time.time()))
    avg_loss = pkg.do_n_fold(params)
    logger.debug("DO_N_FOLD STOP")

    if framework == 'keras':
        # works around this error:
        # https://github.com/tensorflow/tensorflow/issues/3388
        try:
            from keras import backend as K
            K.clear_session()
        except AttributeError:      # theano does not have this function
            pass

    return avg_loss

if __name__ == '__main__':<commit_msg>Disable verbose with new feature<commit_after># tensoflow.__init__ calls _os.path.basename(_sys.argv[0])
# so we need to create a synthetic argv.
import sys
if not hasattr(sys, 'argv'):
    sys.argv  = ['p3b1']

import json
import os
import p3b1
import runner_utils
import socket
import time

node_pid = "%s,%i" % (socket.gethostname(), os.getpid())
print("node,pid: " + node_pid)

logger = None

def get_logger():
    """ Set up logging """
    global logger
    if logger is not None:
        return logger
    import logging, sys
    logger = logging.getLogger(__name__)
    logger.setLevel(logging.DEBUG)
    h = logging.StreamHandler(stream=sys.stdout)
    fmtr = logging.Formatter('%(asctime)s %(name)s %(levelname)-9s %(message)s',
                             datefmt='%Y/%m/%d %H:%M:%S')
    h.setFormatter(fmtr)
    logger.addHandler(h)
    return logger

def run(hyper_parameter_map):

    logger = get_logger()
    framework = hyper_parameter_map['framework']
    logger.debug("IMPORT START " + str(time.time()))
    if framework == 'keras':
        import p3b1_baseline_keras2
        pkg = p3b1_baseline_keras2
    else:
        raise ValueError("Unsupported framework: {}".format(framework))
    logger.debug("IMPORT STOP")

    # params is python dictionary
    params = pkg.initialize_parameters()
    runner_utils.format_params(hyper_parameter_map)

    for k,v in hyper_parameter_map.items():
        #if not k in params:
        #    raise Exception("Parameter '{}' not found in set of valid arguments".format(k))
        params[k] = v

    logger.debug("WRITE_PARAMS START")
    runner_utils.write_params(params, hyper_parameter_map)
    logger.debug("WRITE_PARAMS STOP")
    logger.debug("DO_N_FOLD START " + str(time.time()))
    avg_loss = pkg.do_n_fold(params, verbose=0)
    logger.debug("DO_N_FOLD STOP")

    if framework == 'keras':
        # works around this error:
        # https://github.com/tensorflow/tensorflow/issues/3388
        try:
            from keras import backend as K
            K.clear_session()
        except AttributeError:      # theano does not have this function
            pass

    return avg_loss

if __name__ == '__main__':
{'fraction_non_alphanumeric': 0.11526048870447211, 'fraction_numerical': 0.007607192254495159, 'mean_word_length': 3.055140186915888, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5658356', 'n_tokens_mistral': 1391, 'n_tokens_neox': 1360, 'n_words': 336}