<commit_before>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import argparse
import logging
import os
import random
import re
import signal
import sys
import time
# colorize logging
try:
    import coloredlogs
    colored_logs_present = True
except:
    print("(Install 'coloredlogs' for colored logging)")
    colored_logs_present = False


__all__ = [
    'LOG_FORMAT', 'DATE_FORMAT',                              # constants
    'parser',                                                 # instances
    'exit_handler', 'initialize', 'validate',                 # functions
    'logging', 'os', 'random', 're', 'signal', 'sys', 'time', # modules
]


LOG_FORMAT = '%(asctime)s [%(levelname)s] %(message)s'
DATE_FORMAT = '%H:%M:%S'


def __descr_format(g):
    """
    Help description formatting function to add global documentation variables.

    :param g: globals() dictionary
    :return: the formatted help description
    """
    p = g['__file__']
    p = p[2:] if p.startswith("./") else p
    p = p[:-3] if p.endswith(".py") else p
    s = ''.join([x.capitalize() for x in p.split('-')])
    if '__version__' in g:
        s += " v" + g['__version__']
    for v in ['__author__', '__reference__', '__source__', '__training__']:
        if v in g:
            s += "\n%s: %s" % (v.strip('_').capitalize(), g[v])
            if v == '__author__' and '__email__' in g:
                s += " ({})".format(g['__email__'])
    return s + "\n\n" + g['__doc__'] if '__doc__' in g else s


def __exit_handler(signal=None, frame=None, code=0):
    """
    Exit handler.

    :param signal: signal number
    :param stack: stack frame
    :param code: exit code
    """
    logging.shutdown()
    sys.exit(code)
# bind termination signal (Ctrl+C) to the default exit handler
signal.signal(signal.SIGINT, __exit_handler)


def __updated_exit_handler(signal=None, frame=None, code=0):
    __exit_handler(signal, frame, code)


def exit_handler(glob=None):
    """
    Customized exit handler decorator.

    :param glob: globals() instance from the calling script
    """
    def __wrapper(f):
        global __updated_exit_handler
        def __new_exit_handler(signal=None, frame=None, code=0, *args, **kwargs):
            f(*args, **kwargs)
            __exit_handler(signal, frame, code)
        if glob is not None:
            glob[f] = __new_exit_handler
        __updated_exit_handler = __new_exit_handler
        # this rebinds termination signal (Ctrl+C) to the new exit handler
        signal.signal(signal.SIGINT, __new_exit_handler)
        return __new_exit_handler
    return __wrapper


def initialize(glob, sudo=False):
    """
    Initialization function ; sets up the arguments for the parser and creates a
     logger to be inserted in the input dictionary of global variables from the
     calling script.

    :param glob: globals() instance from the calling script
    :param sudo: if True, require sudo credentials and re-run script with sudo
    """
    global parser
    p = glob['__file__']
    p = p[2:] if p.startswith("./") else p
    e = None if '__examples__' not in glob or len(glob['__examples__']) == 0 \
        else glob['__examples__']
    e = "Usage examples:\n" + '\n'.join(["  python {0} {1}".format(p, x) \
        for x in e]) if e is not None else e
    glob['parser'] = argparse.ArgumentParser(prog=p,
        description=__descr_format(glob), epilog=e,
        formatter_class=argparse.RawTextHelpFormatter)
    for method, args, kwargs in parser.calls:
        getattr(glob['parser'], method)(*args, **kwargs)
    # TODO: adapt debug argument in several ways for supporting -vvv, -vvvv, ...
    try:
        glob['parser'].add_argument("-v", dest="debug", action="store_true",
                                    help="debug verbose level (default: false)")
    except ArgumentError:
        pass  # if debug argument was already passed, just ignore
    glob['args'] = glob['parser'].parse_args()
    if sudo:
        # if not root, restart the script in another process and jump to this
        if os.geteuid() != 0:
            python = [] if glob['__file__'].startswith("./") else ["python"]
            os.execvp("sudo", ["sudo"] + python + sys.argv)
    # configure logging and get the root logger
    glob['args'].verbose = [logging.INFO, logging.DEBUG][glob['args'].debug]
    logging.basicConfig(format=glob['LOG_FORMAT'], datefmt=glob['DATE_FORMAT'],
                        level=glob['args'].verbose)
    glob['logger'] = logging.getLogger()
    if colored_logs_present:
        coloredlogs.DEFAULT_LOG_FORMAT = glob['LOG_FORMAT']<commit_msg>Minor bug fix<commit_after>#!/usr/bin/env python
# -*- coding: UTF-8 -*-
import argparse
import logging
import os
import random
import re
import signal
import sys
import time
# colorize logging
try:
    import coloredlogs
    colored_logs_present = True
except:
    print("(Install 'coloredlogs' for colored logging)")
    colored_logs_present = False


__all__ = [
    'LOG_FORMAT', 'DATE_FORMAT',                              # constants
    'parser',                                                 # instances
    'exit_handler', 'initialize', 'validate',                 # functions
    'logging', 'os', 'random', 're', 'signal', 'sys', 'time', # modules
]


LOG_FORMAT = '%(asctime)s [%(levelname)s] %(message)s'
DATE_FORMAT = '%H:%M:%S'


def __descr_format(g):
    """
    Help description formatting function to add global documentation variables.

    :param g: globals() dictionary
    :return: the formatted help description
    """
    p = g['__file__']
    p = p[2:] if p.startswith("./") else p
    p = p[:-3] if p.endswith(".py") else p
    s = ''.join([x.capitalize() for x in p.split('-')])
    if '__version__' in g:
        s += " v" + g['__version__']
    for v in ['__author__', '__reference__', '__source__', '__training__']:
        if v in g:
            s += "\n%s: %s" % (v.strip('_').capitalize(), g[v])
            if v == '__author__' and '__email__' in g:
                s += " ({})".format(g['__email__'])
    return s + "\n\n" + g['__doc__'] if '__doc__' in g else s


def __exit_handler(signal=None, frame=None, code=0):
    """
    Exit handler.

    :param signal: signal number
    :param stack: stack frame
    :param code: exit code
    """
    logging.shutdown()
    sys.exit(code)
# bind termination signal (Ctrl+C) to the default exit handler
signal.signal(signal.SIGINT, __exit_handler)


def __updated_exit_handler(signal=None, frame=None, code=0):
    __exit_handler(signal, frame, code)


def exit_handler(glob=None):
    """
    Customized exit handler decorator.

    :param glob: globals() instance from the calling script
    """
    def __wrapper(f):
        global __updated_exit_handler
        def __new_exit_handler(signal=None, frame=None, code=0, *args, **kwargs):
            f(*args, **kwargs)
            __exit_handler(signal, frame, code)
        if glob is not None:
            glob[f] = __new_exit_handler
        __updated_exit_handler = __new_exit_handler
        # this rebinds termination signal (Ctrl+C) to the new exit handler
        signal.signal(signal.SIGINT, __new_exit_handler)
        return __new_exit_handler
    return __wrapper


def initialize(glob, sudo=False):
    """
    Initialization function ; sets up the arguments for the parser and creates a
     logger to be inserted in the input dictionary of global variables from the
     calling script.

    :param glob: globals() instance from the calling script
    :param sudo: if True, require sudo credentials and re-run script with sudo
    """
    global parser
    p = glob['__file__']
    p = p[2:] if p.startswith("./") else p
    e = None if '__examples__' not in glob or len(glob['__examples__']) == 0 \
        else glob['__examples__']
    e = "Usage examples:\n" + '\n'.join(["  python {0} {1}".format(p, x) \
        for x in e]) if e is not None else e
    glob['parser'] = argparse.ArgumentParser(prog=p,
        description=__descr_format(glob), epilog=e,
        formatter_class=argparse.RawTextHelpFormatter)
    for method, args, kwargs in parser.calls:
        getattr(glob['parser'], method)(*args, **kwargs)
    # TODO: adapt debug argument in several ways for supporting -vvv, -vvvv, ...
    try:
        glob['parser'].add_argument("-v", dest="debug", action="store_true",
                                    help="debug verbose level (default: false)")
    except argparse.ArgumentError:
        pass  # if debug argument was already passed, just ignore
    glob['args'] = glob['parser'].parse_args()
    if sudo:
        # if not root, restart the script in another process and jump to this
        if os.geteuid() != 0:
            python = [] if glob['__file__'].startswith("./") else ["python"]
            os.execvp("sudo", ["sudo"] + python + sys.argv)
    # configure logging and get the root logger
    glob['args'].verbose = [logging.INFO, logging.DEBUG][glob['args'].debug]
    logging.basicConfig(format=glob['LOG_FORMAT'], datefmt=glob['DATE_FORMAT'],
                        level=glob['args'].verbose)
    glob['logger'] = logging.getLogger()
    if colored_logs_present:
        coloredlogs.DEFAULT_LOG_FORMAT = glob['LOG_FORMAT']
{'fraction_non_alphanumeric': 0.1309071156979902, 'fraction_numerical': 0.002390005431830527, 'mean_word_length': 2.788477366255144, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4517039', 'n_tokens_mistral': 2815, 'n_tokens_neox': 2738, 'n_words': 835}