<commit_before>"Generic context manager for separating code execution across processes."

import inspect
import os
from multiprocessing.connection import Pipe
import sys
import threading
import traceback


class WithParentSkip(object):
    """Context manager separating code execution across parent/child processes.

    This is done by forking and doing some magic on the stack so the contents
    of the context are executed only on the forked child and the parent.
    Exceptions are pickled and passed back to the parent.
    """
    def __init__(self):
        self.__trace_lock = threading.Lock()
        self.__orig_sys_trace = None
        self.__orig_trace_funcs = {}
        self.__injected_trace_funcs = {}
        self.__pipe = None
        self.childpid = None

    def parent_setup(self):
        """Initialization for parent process."""
        pass

    def child_setup(self):
        """Initialization for child process."""
        raise NotImplementedError

    def cleanup(self):
        """Parent process clean up on termination of the child."""
        raise NotImplementedError

    def exception_cleanup(self):
        """Parent process clean up after the child throws an exception."""
        self.cleanup()

    def __enter__(self):
        parent_pipe, child_pipe = Pipe()
        childpid = os.fork()

        if childpid != 0:
            self.parent_setup()
            self.childpid = childpid
            self.__pipe = parent_pipe
            frame = self.__get_context_frame()
            self.__inject_trace_func(frame, self.__exit_context)

            return self

        else:
            self.__pipe = child_pipe

            try:
                self.child_setup()

            # pylint: disable=W0703
            # need to catch all exceptions here since we are passing them to
            # the parent process
            except Exception as ex:
                ex.__traceback_list__ = traceback.format_exc()
                self.__pipe.send(ex)
                try:
                    self.__pipe.send(SystemExit())
                except (BrokenPipeError if sys.hexversion >= 0x03030000 else OSError, IOError):  # pylint: disable=E0602
                    pass
                os._exit(0)  # pylint: disable=W0212
                # we don't want SystemExit being caught here

            return self

    def __exit__(self, exc_type, exc_value, exc_traceback):
        if exc_type is self.ParentException:
            try:
                exception = self.__pipe.recv()
            except EOFError as e:
                exception = SystemExit(e)

            if not isinstance(exception, SystemExit):
                os.waitpid(self.childpid, 0)
                self.exception_cleanup()
                sys.excepthook = self.__excepthook
                raise exception

        elif exc_value is not None:
            # Unfortunately, traceback objects can't be pickled so the relevant
            # traceback from the code executing within the chroot context is
            # placed in the __traceback_list__ attribute and printed by a
            # custom exception hook.
            exc_value.__traceback_list__ = traceback.format_exc()
            self.__pipe.send(exc_value)

        if self.childpid is None:
            try:
                self.__pipe.send(SystemExit())
            except (BrokenPipeError if sys.hexversion >= 0x03030000 else OSError, IOError):  # pylint: disable=E0602
                pass
            os._exit(0)  # pylint: disable=W0212

        # wait for child process to exit
        os.waitpid(self.childpid, 0)
        self.cleanup()

        return True

    @staticmethod
    def __excepthook(_exc_type, exc_value, exc_traceback):<commit_msg>specifically check for broken pipe errors<commit_after>"Generic context manager for separating code execution across processes."

import errno
import inspect
import os
from multiprocessing.connection import Pipe
import sys
import threading
import traceback


class WithParentSkip(object):
    """Context manager separating code execution across parent/child processes.

    This is done by forking and doing some magic on the stack so the contents
    of the context are executed only on the forked child and the parent.
    Exceptions are pickled and passed back to the parent.
    """
    def __init__(self):
        self.__trace_lock = threading.Lock()
        self.__orig_sys_trace = None
        self.__orig_trace_funcs = {}
        self.__injected_trace_funcs = {}
        self.__pipe = None
        self.childpid = None

    def parent_setup(self):
        """Initialization for parent process."""
        pass

    def child_setup(self):
        """Initialization for child process."""
        raise NotImplementedError

    def cleanup(self):
        """Parent process clean up on termination of the child."""
        raise NotImplementedError

    def exception_cleanup(self):
        """Parent process clean up after the child throws an exception."""
        self.cleanup()

    def __enter__(self):
        parent_pipe, child_pipe = Pipe()
        childpid = os.fork()

        if childpid != 0:
            self.parent_setup()
            self.childpid = childpid
            self.__pipe = parent_pipe
            frame = self.__get_context_frame()
            self.__inject_trace_func(frame, self.__exit_context)

            return self

        else:
            self.__pipe = child_pipe

            try:
                self.child_setup()

            # pylint: disable=W0703
            # need to catch all exceptions here since we are passing them to
            # the parent process
            except Exception as ex:
                ex.__traceback_list__ = traceback.format_exc()
                self.__pipe.send(ex)
                try:
                    self.__pipe.send(SystemExit())
                except (BrokenPipeError if sys.hexversion >= 0x03030000  # pylint: disable=E0602
                        else OSError, IOError) as ex:
                    if ex.errno in (errno.EPIPE, errno.ESHUTDOWN):
                        pass
                    else:
                        raise
                os._exit(0)  # pylint: disable=W0212
                # we don't want SystemExit being caught here

            return self

    def __exit__(self, exc_type, exc_value, exc_traceback):
        if exc_type is self.ParentException:
            try:
                exception = self.__pipe.recv()
            except EOFError as e:
                exception = SystemExit(e)

            if not isinstance(exception, SystemExit):
                os.waitpid(self.childpid, 0)
                self.exception_cleanup()
                sys.excepthook = self.__excepthook
                raise exception

        elif exc_value is not None:
            # Unfortunately, traceback objects can't be pickled so the relevant
            # traceback from the code executing within the chroot context is
            # placed in the __traceback_list__ attribute and printed by a
            # custom exception hook.
            exc_value.__traceback_list__ = traceback.format_exc()
            self.__pipe.send(exc_value)

        if self.childpid is None:
            try:
                self.__pipe.send(SystemExit())
            except (BrokenPipeError if sys.hexversion >= 0x03030000  # pylint: disable=E0602
                    else OSError, IOError) as ex:
                if ex.errno in (errno.EPIPE, errno.ESHUTDOWN):
                    pass
                else:
                    raise
            os._exit(0)  # pylint: disable=W0212

        # wait for child process to exit
        os.waitpid(self.childpid, 0)
        self.cleanup()

        return True

    @staticmethod
    def __excepthook(_exc_type, exc_value, exc_traceback):
{'fraction_non_alphanumeric': 0.06679511881824021, 'fraction_numerical': 0.011046885035324342, 'mean_word_length': 1.8730627306273062, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3252578', 'n_tokens_mistral': 2103, 'n_tokens_neox': 1934, 'n_words': 628}