<commit_before>"""
Logfile setup
"""

from __future__ import absolute_import, division, print_function

import os
import sys
import logging
import logging.handlers

setlevel = {
  'CRITICAL': logging.CRITICAL, # execution cannot continue
  'FATAL': logging.CRITICAL,
  'ERROR': logging.ERROR, # something is wrong, but try to continue
  'WARNING': logging.WARNING, # non-ideal behavior, important event
  'WARN': logging.WARNING,
  'INFO': logging.INFO, # initial debug information
  'DEBUG': logging.DEBUG # the things no one wants to see
  }

host = os.uname()[1].split(".")[0]

def setlogger(loglevel='INFO', logfile='sys.stdout', logsize=2**24, lognum=4):
    """Add an output to the root logger"""
    logformat='%(asctime)s %(levelname)s %(name)s : %(message)s'

    rootLogger = logging.getLogger()

    if logfile.strip() != 'sys.stdout':
        if not logfile.startswith('/'):
            if 'I3PROD' in os.environ:
                logfile = os.path.expanduser(os.path.expandvars(
                            os.path.join('$I3PROD', 'var', 'log', logfile)))
            else:
                logfile = os.path.join(os.getcwd(), 'log', host, logfile)
        if not os.path.exists(os.path.dirname(logfile)):
            os.makedirs(os.path.dirname(logfile))
        fileHandler = logging.handlers.RotatingFileHandler(logfile, 'a',
                                                           logsize, lognum)
        formatter = logging.Formatter(logformat)
        fileHandler.setFormatter(formatter)
        rootLogger.addHandler(fileHandler)
        for handler in rootLogger.handlers:
            if handler != fileHandler:
                rootLogger.removeHandler(handler)
        rootLogger.info('fileHandler used')
    else:
        logging.basicConfig(format=logformat)
        rootLogger.info('basicConfig used')
<commit_msg>forgot to set the level in the last commit<commit_after>"""
Logfile setup
"""

from __future__ import absolute_import, division, print_function

import os
import sys
import logging
import logging.handlers

setlevel = {
  'CRITICAL': logging.CRITICAL, # execution cannot continue
  'FATAL': logging.CRITICAL,
  'ERROR': logging.ERROR, # something is wrong, but try to continue
  'WARNING': logging.WARNING, # non-ideal behavior, important event
  'WARN': logging.WARNING,
  'INFO': logging.INFO, # initial debug information
  'DEBUG': logging.DEBUG # the things no one wants to see
  }

host = os.uname()[1].split(".")[0]

def setlogger(loglevel='INFO', logfile='sys.stdout', logsize=2**24, lognum=4):
    """Add an output to the root logger"""
    logformat='%(asctime)s %(levelname)s %(name)s : %(message)s'

    rootLogger = logging.getLogger()

    if logfile.strip() != 'sys.stdout':
        if not logfile.startswith('/'):
            if 'I3PROD' in os.environ:
                logfile = os.path.expanduser(os.path.expandvars(
                            os.path.join('$I3PROD', 'var', 'log', logfile)))
            else:
                logfile = os.path.join(os.getcwd(), 'log', host, logfile)
        if not os.path.exists(os.path.dirname(logfile)):
            os.makedirs(os.path.dirname(logfile))
        fileHandler = logging.handlers.RotatingFileHandler(logfile, 'a',
                                                           logsize, lognum)
        formatter = logging.Formatter(logformat)
        fileHandler.setFormatter(formatter)
        rootLogger.addHandler(fileHandler)
        for handler in rootLogger.handlers:
            if handler != fileHandler:
                rootLogger.removeHandler(handler)
        rootLogger.setLevel(setlevel[loglevel.upper()])
        rootLogger.info('fileHandler used')
    else:
        logging.basicConfig(format=logformat, level=setlevel[loglevel.upper()])
        rootLogger.info('basicConfig used')

{'fraction_non_alphanumeric': 0.12496711391739015, 'fraction_numerical': 0.004209418574059458, 'mean_word_length': 2.764356435643564, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2812048', 'n_tokens_mistral': 1122, 'n_tokens_neox': 1107, 'n_words': 262}