<commit_before># cmd.py
# Copyright (C) 2008, 2009 <NAME> (<EMAIL>) and contributors
#
# This module is part of GitPython and is released under
# the BSD License: http://www.opensource.org/licenses/bsd-license.php

import os
import sys
import select
import logging
import threading
from subprocess import (
    call,
    Popen,
    PIPE
)


from .util import (
    LazyMixin,
    stream_copy,
    WaitGroup
)
from .exc import GitCommandError
from git.compat import (
    string_types,
    defenc,
    PY3,
    # just to satisfy flake8 on py3
    unicode
)

execute_kwargs = ('istream', 'with_keep_cwd', 'with_extended_output',
                  'with_exceptions', 'as_process', 'stdout_as_string',
                  'output_stream')

log = logging.getLogger('git.cmd')

__all__ = ('Git', )

if sys.platform != 'win32':
    WindowsError = OSError


# ==============================================================================
## @name Utilities
# ------------------------------------------------------------------------------
# Documentation
## @{

def handle_process_output(process, stdout_handler, stderr_handler, finalizer):
    """Registers for notifications to lean that process output is ready to read, and dispatches lines to
    the respective line handlers. We are able to handle carriage returns in case progress is sent by that
    mean. For performance reasons, we only apply this to stderr.
    This function returns once the finalizer returns
    :return: result of finalizer
    :param process: subprocess.Popen instance
    :param stdout_handler: f(stdout_line_string), or None
    :param stderr_hanlder: f(stderr_line_string), or None
    :param finalizer: f(proc) - wait for proc to finish"""
    def read_line_fast(stream):
        return stream.readline()

    def read_line_slow(stream):
        line = b''
        while True:
            char = stream.read(1)       # reads individual single byte strings
            if not char:
                break

            if char in (b'\r', b'\n') and line:
                break
            else:
                line += char
            # END process parsed line
        # END while file is not done reading
        return line
    # end

    def dispatch_line(fno):
            stream, handler, readline = fdmap[fno]
            # this can possibly block for a while, but since we wake-up with at least one or more lines to handle,
            # we are good ...
            line = readline(stream).decode(defenc)
            if line and handler:
                try:
                    handler(line)
                except Exception:
                    # Keep reading, have to pump the lines empty nontheless
                    log.error("Line handler exception on line: %s", line, exc_info=True)
                # end
            return line
        # end dispatch helper
    # end

    def deplete_buffer(fno, wg=None):
        while True:
            line = dispatch_line(fno)
            if not line:
                break
        # end deplete buffer
        if wg:
            wg.done()
    # end

    fdmap = {process.stdout.fileno(): (process.stdout, stdout_handler, read_line_fast),
             process.stderr.fileno(): (process.stderr, stderr_handler, read_line_slow)}

    if hasattr(select, 'poll'):
        # poll is preferred, as select is limited to file handles up to 1024 ... . This could otherwise be
        # an issue for us, as it matters how many handles or own process has
        poll = select.poll()
        READ_ONLY = select.POLLIN | select.POLLPRI | select.POLLHUP | select.POLLERR
        CLOSED = select.POLLHUP | select.POLLERR

        poll.register(process.stdout, READ_ONLY)
        poll.register(process.stderr, READ_ONLY)

        closed_streams = set()
        while True:
            # no timeout
            poll_result = poll.poll()
            for fd, result in poll_result:
                if result & CLOSED:
                    closed_streams.add(fd)
                else:
                    dispatch_line(fd)
                # end handle closed stream
            # end for each poll-result tuple

            if len(closed_streams) == len(fdmap):
                break
            # end its all done
        # end endless loop

        # Depelete all remaining buffers
        for fno in fdmap.keys():
            deplete_buffer(fno)
        # end for each file handle
    else:
        # Oh ... probably we are on windows. select.select() can only handle sockets, we have files
        # The only reliable way to do this now is to use threads and wait for both to finish
        # Since the finalizer is expected to wait, we don't have to introduce our own wait primitive
        # NO: It's not enough unfortunately, and we will have to sync the threads
        wg = WaitGroup()
        for fno in fdmap.keys():
            wg.add(1)
            t = threading.Thread(target=lambda: deplete_buffer(fno, wg))
            t.start()
        # end
        # NOTE: Just joining threads can possibly fail as there is a gap between .start() and when it's
        # actually started, which could make the wait() call to just return because the thread is not yet
        # active
        wg.wait()
    # end

    return finalizer(process)


def dashify(string):
    return string.replace('_', '-')

## -- End Utilities -- @}


class Git(LazyMixin):

    """
    The Git class manages communication with the Git binary.

    It provides a convenient interface to calling the Git binary, such as in::
<commit_msg>An attempt to help the threaded reading to not show spurious errors anymore.<commit_after># cmd.py
# Copyright (C) 2008, 2009 <NAME> (<EMAIL>) and contributors
#
# This module is part of GitPython and is released under
# the BSD License: http://www.opensource.org/licenses/bsd-license.php

import os
import sys
import select
import logging
import threading
from subprocess import (
    call,
    Popen,
    PIPE
)


from .util import (
    LazyMixin,
    stream_copy,
    WaitGroup
)
from .exc import GitCommandError
from git.compat import (
    string_types,
    defenc,
    PY3,
    # just to satisfy flake8 on py3
    unicode
)

execute_kwargs = ('istream', 'with_keep_cwd', 'with_extended_output',
                  'with_exceptions', 'as_process', 'stdout_as_string',
                  'output_stream')

log = logging.getLogger('git.cmd')

__all__ = ('Git', )

if sys.platform != 'win32':
    WindowsError = OSError


# ==============================================================================
## @name Utilities
# ------------------------------------------------------------------------------
# Documentation
## @{

def handle_process_output(process, stdout_handler, stderr_handler, finalizer):
    """Registers for notifications to lean that process output is ready to read, and dispatches lines to
    the respective line handlers. We are able to handle carriage returns in case progress is sent by that
    mean. For performance reasons, we only apply this to stderr.
    This function returns once the finalizer returns
    :return: result of finalizer
    :param process: subprocess.Popen instance
    :param stdout_handler: f(stdout_line_string), or None
    :param stderr_hanlder: f(stderr_line_string), or None
    :param finalizer: f(proc) - wait for proc to finish"""
    def read_line_fast(stream):
        return stream.readline()

    def read_line_slow(stream):
        line = b''
        while True:
            char = stream.read(1)       # reads individual single byte strings
            if not char:
                break

            if char in (b'\r', b'\n') and line:
                break
            else:
                line += char
            # END process parsed line
        # END while file is not done reading
        return line
    # end

    def dispatch_line(stream, handler, readline):
            # this can possibly block for a while, but since we wake-up with at least one or more lines to handle,
            # we are good ...
            line = readline(stream).decode(defenc)
            if line and handler:
                try:
                    handler(line)
                except Exception:
                    # Keep reading, have to pump the lines empty nontheless
                    log.error("Line handler exception on line: %s", line, exc_info=True)
                # end
            return line
        # end dispatch helper
    # end

    def deplete_buffer(stream, handler, readline, wg=None):
        while True:
            line = dispatch_line(stream, handler, readline)
            if not line:
                break
        # end deplete buffer
        if wg:
            wg.done()
    # end

    fdmap = {process.stdout.fileno(): (process.stdout, stdout_handler, read_line_fast),
             process.stderr.fileno(): (process.stderr, stderr_handler, read_line_slow)}

    if hasattr(select, 'poll'):
        # poll is preferred, as select is limited to file handles up to 1024 ... . This could otherwise be
        # an issue for us, as it matters how many handles or own process has
        poll = select.poll()
        READ_ONLY = select.POLLIN | select.POLLPRI | select.POLLHUP | select.POLLERR
        CLOSED = select.POLLHUP | select.POLLERR

        poll.register(process.stdout, READ_ONLY)
        poll.register(process.stderr, READ_ONLY)

        closed_streams = set()
        while True:
            # no timeout
            poll_result = poll.poll()
            for fd, result in poll_result:
                if result & CLOSED:
                    closed_streams.add(fd)
                else:
                    dispatch_line(*fdmap[fd])
                # end handle closed stream
            # end for each poll-result tuple

            if len(closed_streams) == len(fdmap):
                break
            # end its all done
        # end endless loop

        # Depelete all remaining buffers
        for stream, handler, readline in fdmap.values():
            deplete_buffer(stream, handler, readline)
        # end for each file handle
    else:
        # Oh ... probably we are on windows. select.select() can only handle sockets, we have files
        # The only reliable way to do this now is to use threads and wait for both to finish
        # Since the finalizer is expected to wait, we don't have to introduce our own wait primitive
        # NO: It's not enough unfortunately, and we will have to sync the threads
        wg = WaitGroup()
        for fno in fdmap.keys():
            wg.add(1)
            stream, handler, readline = fdmap[fno]
            t = threading.Thread(target=lambda: deplete_buffer(stream, handler, readline, wg))
            t.start()
        # end
        # NOTE: Just joining threads can possibly fail as there is a gap between .start() and when it's
        # actually started, which could make the wait() call to just return because the thread is not yet
        # active
        wg.wait()
    # end

    return finalizer(process)


def dashify(string):
    return string.replace('_', '-')

## -- End Utilities -- @}


class Git(LazyMixin):

    """
    The Git class manages communication with the Git binary.

    It provides a convenient interface to calling the Git binary, such as in::

{'fraction_non_alphanumeric': 0.10361681329423265, 'fraction_numerical': 0.003376877277170532, 'mean_word_length': 2.0317887931034484, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3126293', 'n_tokens_mistral': 3074, 'n_tokens_neox': 2919, 'n_words': 1116}