<commit_before>
"""
    ForkedFunc provides a way to run a function in a forked process
    and get at its return value, stdout and stderr output as well
    as signals and exitstatusus.
"""

import py
import os
import sys
import marshal



class HookMixin(object):
    _on_start = list()
    _on_exit = list()

    @classmethod
    def register_on_start(cls, callback):
        cls._on_start.append(callback)

    @classmethod
    def register_on_exit(self, callback):
        self._on_exit.append(callback)

    def _run_on_start(self):
        self._run_callbacks(self._on_start)

    def _run_on_exit(self):
        self._run_callbacks(self._on_exit)

    def _run_callbacks(self, callbacks):
        for callback in callbacks:
            callback(self)


class ForkedFunc(HookMixin):
    EXITSTATUS_EXCEPTION = 3


    def __init__(self, fun, args=None, kwargs=None, nice_level=0):
        if args is None:
            args = []
        if kwargs is None:
            kwargs = {}
        self.fun = fun
        self.args = args
        self.kwargs = kwargs
        self.tempdir = tempdir = py.path.local.mkdtemp()
        self.RETVAL = tempdir.ensure('retval')
        self.STDOUT = tempdir.ensure('stdout')
        self.STDERR = tempdir.ensure('stderr')

        pid = os.fork()
        if pid:  # in parent process
            self.pid = pid
        else:  # in child process
            self.pid = None
            self._child(nice_level)

    def _child(self, nice_level):
        # right now we need to call a function, but first we need to
        # map all IO that might happen
        sys.stdout = stdout = open(str(self.STDOUT), "w", 0)
        fdstdout = stdout.fileno()
        if fdstdout != 1:
            os.dup2(fdstdout, 1)
        sys.stderr = stderr = open(str(self.STDERR), "w", 0)
        fdstderr = stderr.fileno()
        if fdstderr != 2:
            os.dup2(fdstderr, 2)
        retvalf = self.RETVAL.open("wb")
        EXITSTATUS = 0
        try:
            if nice_level:
                os.nice(nice_level)
            try:
                self._run_on_start()
                retval = self.fun(*self.args, **self.kwargs)
                retvalf.write(marshal.dumps(retval))
                self._run_on_exit()
            except:
                excinfo = py.code.ExceptionInfo()
                stderr.write(excinfo.exconly())
                EXITSTATUS = self.EXITSTATUS_EXCEPTION
        finally:
            stdout.close()
            stderr.close()
            retvalf.close()
        os.close(1)
        os.close(2)
        os._exit(EXITSTATUS)

    def waitfinish(self, waiter=os.waitpid):
        pid, systemstatus = waiter(self.pid, 0)
        if systemstatus:
            if os.WIFSIGNALED(systemstatus):
                exitstatus = os.WTERMSIG(systemstatus) + 128
            else:
                exitstatus = os.WEXITSTATUS(systemstatus)
        else:
            exitstatus = 0
        signal = systemstatus & 0x7f
        if not exitstatus and not signal:
            retval = self.RETVAL.open('rb')
            try:
                retval_data = retval.read()
            finally:
                retval.close()
            retval = marshal.loads(retval_data)
        else:
            retval = None
        stdout = self.STDOUT.read()
        stderr = self.STDERR.read()
        self._removetemp()
        return Result(exitstatus, signal, retval, stdout, stderr)

    def _removetemp(self):
        if self.tempdir.check():
            self.tempdir.remove()

    def __del__(self):
        if self.pid is not None:  # only clean up in main process
            self._removetemp()



class Result(object):<commit_msg>fix unbuffered I/O on Python 3<commit_after>
"""
    ForkedFunc provides a way to run a function in a forked process
    and get at its return value, stdout and stderr output as well
    as signals and exitstatusus.
"""

import py
import os
import sys
import marshal


class HookMixin(object):
    _on_start = list()
    _on_exit = list()

    @classmethod
    def register_on_start(cls, callback):
        cls._on_start.append(callback)

    @classmethod
    def register_on_exit(self, callback):
        self._on_exit.append(callback)

    def _run_on_start(self):
        self._run_callbacks(self._on_start)

    def _run_on_exit(self):
        self._run_callbacks(self._on_exit)

    def _run_callbacks(self, callbacks):
        for callback in callbacks:
            callback(self)


class ForkedFunc(HookMixin):
    EXITSTATUS_EXCEPTION = 3


    def __init__(self, fun, args=None, kwargs=None, nice_level=0):
        if args is None:
            args = []
        if kwargs is None:
            kwargs = {}
        self.fun = fun
        self.args = args
        self.kwargs = kwargs
        self.tempdir = tempdir = py.path.local.mkdtemp()
        self.RETVAL = tempdir.ensure('retval')
        self.STDOUT = tempdir.ensure('stdout')
        self.STDERR = tempdir.ensure('stderr')

        pid = os.fork()
        if pid:  # in parent process
            self.pid = pid
        else:  # in child process
            self.pid = None
            self._child(nice_level)

    def _child(self, nice_level):
        # right now we need to call a function, but first we need to
        # map all IO that might happen
        sys.stdout = stdout = open(str(self.STDOUT), "wb", 0)
        fdstdout = stdout.fileno()
        if fdstdout != 1:
            os.dup2(fdstdout, 1)
        sys.stderr = stderr = open(str(self.STDERR), "wb", 0)
        fdstderr = stderr.fileno()
        if fdstderr != 2:
            os.dup2(fdstderr, 2)
        retvalf = self.RETVAL.open("wb")
        EXITSTATUS = 0
        try:
            if nice_level:
                os.nice(nice_level)
            try:
                self._run_on_start()
                retval = self.fun(*self.args, **self.kwargs)
                retvalf.write(marshal.dumps(retval))
                self._run_on_exit()
            except:
                excinfo = py.code.ExceptionInfo()
                stderr.write(excinfo.exconly())
                EXITSTATUS = self.EXITSTATUS_EXCEPTION
        finally:
            stdout.close()
            stderr.close()
            retvalf.close()
        os.close(1)
        os.close(2)
        os._exit(EXITSTATUS)

    def waitfinish(self, waiter=os.waitpid):
        pid, systemstatus = waiter(self.pid, 0)
        if systemstatus:
            if os.WIFSIGNALED(systemstatus):
                exitstatus = os.WTERMSIG(systemstatus) + 128
            else:
                exitstatus = os.WEXITSTATUS(systemstatus)
        else:
            exitstatus = 0
        signal = systemstatus & 0x7f
        if not exitstatus and not signal:
            retval = self.RETVAL.open('rb')
            try:
                retval_data = retval.read()
            finally:
                retval.close()
            retval = marshal.loads(retval_data)
        else:
            retval = None
        stdout = self.STDOUT.read()
        stderr = self.STDERR.read()
        self._removetemp()
        return Result(exitstatus, signal, retval, stdout, stderr)

    def _removetemp(self):
        if self.tempdir.check():
            self.tempdir.remove()

    def __del__(self):
        if self.pid is not None:  # only clean up in main process
            self._removetemp()


class Result(object):
{'fraction_non_alphanumeric': 0.09530931339225017, 'fraction_numerical': 0.00557443915703603, 'mean_word_length': 1.7832009080590239, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1042876', 'n_tokens_mistral': 2273, 'n_tokens_neox': 2176, 'n_words': 534}