<commit_before>"""
Copied code from http://stackoverflow.com/questions/6728236/exception-thrown-in-multiprocessing-pool-not-detected
This entire script is a workaround for python multiprocessing module inability to bubble exception stack trace
    to the parent process from a child process.
Explicitly sets multiprocessing log level to WARN.
"""
import traceback
from multiprocessing.pool import Pool
import multiprocessing
import logging

LOGGER = logging.getLogger(__name__) #multiprocessing.get_logger()
#LOGGER.setLevel(logging.WARN)
#LOGGER.propagate = True

# Shortcut to multiprocessing's logger
def error(msg, *args):
    """
    use multiprocessing's logger to log an ERROR
    """
    print "here"
    return LOGGER.error(msg, *args)


class LogExceptions(object):
    """
    Bubble exception stack trace from child process to parent
    """
    def __init__(self, callable):
        self.__callable = callable
        return

    def __call__(self, *args, **kwargs):
        try:
            result = self.__callable(*args, **kwargs)

        except Exception as e:
            # Here we add some debugging help. If multiprocessing's
            # debugging is on, it will arrange to log the traceback
            error(traceback.format_exc())
            # Re-raise the original exception so the Pool worker can
            # clean up
            raise Exception

        # It was fine, give a normal answer
        return result
    pass


class LoggingPool(Pool):
    """
    Wrapper class for multiprocessing.Pool that logs that full exception stack trace from a child process.<commit_msg>Explicitly set LoggingPool logging to WARN<commit_after>"""
Copied code from http://stackoverflow.com/questions/6728236/exception-thrown-in-multiprocessing-pool-not-detected
This entire script is a workaround for python multiprocessing module inability to bubble exception stack trace
    to the parent process from a child process.
Explicitly sets multiprocessing log level to WARN.
"""
import traceback
from multiprocessing.pool import Pool
import logging

LOGGER = logging.getLogger(__name__)
LOGGER.setLevel(logging.WARN)


# Shortcut to multiprocessing's logger
def error(msg, *args):
    """
    use multiprocessing's logger to log an ERROR
    """
    return LOGGER.error(msg, *args)


class LogExceptions(object):
    """
    Bubble exception stack trace from child process to parent
    """
    def __init__(self, callable):
        self.__callable = callable
        return

    def __call__(self, *args, **kwargs):
        try:
            result = self.__callable(*args, **kwargs)

        except Exception as e:
            # Here we add some debugging help. If multiprocessing's
            # debugging is on, it will arrange to log the traceback
            error(traceback.format_exc())
            # Re-raise the original exception so the Pool worker can
            # clean up
            raise Exception

        # It was fine, give a normal answer
        return result
    pass


class LoggingPool(Pool):
    """
    Wrapper class for multiprocessing.Pool that logs that full exception stack trace from a child process.
{'fraction_non_alphanumeric': 0.07183725365543547, 'fraction_numerical': 0.004450095359186269, 'mean_word_length': 2.9785082174462705, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4778179', 'n_tokens_mistral': 866, 'n_tokens_neox': 831, 'n_words': 315}