<commit_before># utils.py
# Copyright (C) 2008, 2009 <NAME> (<EMAIL>) and contributors
#
# This module is part of GitPython and is released under
# the BSD License: http://www.opensource.org/licenses/bsd-license.php

import os
import re
import sys
import time
import stat
import shutil
import platform
import getpass
import threading

# NOTE:  Some of the unused imports might be used/imported by others.
# Handle once test-cases are back up and running.
from .exc import InvalidGitRepositoryError

from .compat import (
    MAXSIZE,
    defenc,
    PY3
)

# Most of these are unused here, but are for use by git-python modules so these
# don't see gitdb all the time. Flake of course doesn't like it.
from gitdb.util import (  # NOQA
    make_sha,
    LockedFD,
    file_contents_ro,
    LazyMixin,
    to_hex_sha,
    to_bin_sha
)

__all__ = ("stream_copy", "join_path", "to_native_path_windows", "to_native_path_linux",
           "join_path_native", "Stats", "IndexFileSHA1Writer", "Iterable", "IterableList",
           "BlockingLockFile", "LockFile", 'Actor', 'get_user_id', 'assure_directory_exists',
           'RemoteProgress', 'rmtree', 'WaitGroup', 'unbare_repo')

#{ Utility Methods


def unbare_repo(func):
    """Methods with this decorator raise InvalidGitRepositoryError if they
    encounter a bare repository"""

    def wrapper(self, *args, **kwargs):
        if self.repo.bare:
            raise InvalidGitRepositoryError("Method '%s' cannot operate on bare repositories" % func.__name__)
        # END bare method
        return func(self, *args, **kwargs)
    # END wrapper
    wrapper.__name__ = func.__name__
    return wrapper


def rmtree(path):
    """Remove the given recursively.

    :note: we use shutil rmtree but adjust its behaviour to see whether files that
        couldn't be deleted are read-only. Windows will not remove them in that case"""
    def onerror(func, path, exc_info):
        if not os.access(path, os.W_OK):
            # Is the error an access error ?
            os.chmod(path, stat.S_IWUSR)
            func(path)
        else:
            raise
    # END end onerror
    return shutil.rmtree(path, False, onerror)


def stream_copy(source, destination, chunk_size=512 * 1024):
    """Copy all data from the source stream into the destination stream in chunks
    of size chunk_size

    :return: amount of bytes written"""
    br = 0
    while True:
        chunk = source.read(chunk_size)
        destination.write(chunk)
        br += len(chunk)
        if len(chunk) < chunk_size:
            break
    # END reading output stream
    return br


def join_path(a, *p):
    """Join path tokens together similar to os.path.join, but always use
    '/' instead of possibly '\' on windows."""
    path = a
    for b in p:
        if len(b) == 0:
            continue
        if b.startswith('/'):
            path += b[1:]
        elif path == '' or path.endswith('/'):
            path += b
        else:
            path += '/' + b
    # END for each path token to add
    return path


if sys.platform.startswith('win'):
    def to_native_path_windows(path):
        return path.replace('/', '\\')

    def to_native_path_linux(path):
        return path.replace('\\', '/')

    to_native_path = to_native_path_windows
else:
    # no need for any work on linux
    def to_native_path_linux(path):
        return path
    to_native_path = to_native_path_linux


def join_path_native(a, *p):
    """
    As join path, but makes sure an OS native path is returned. This is only
        needed to play it safe on my dear windows and to assure nice paths that only
        use '\'"""
    return to_native_path(join_path(a, *p))


def assure_directory_exists(path, is_file=False):
    """Assure that the directory pointed to by path exists.

    :param is_file: If True, path is assumed to be a file and handled correctly.
        Otherwise it must be a directory
    :return: True if the directory was created, False if it already existed"""
    if is_file:
        path = os.path.dirname(path)
    # END handle file
    if not os.path.isdir(path):
        os.makedirs(path)
        return True
    return False


def get_user_id():
    """:return: string identifying the currently active system user as name@node"""
    return "%s@%s" % (getpass.getuser(), platform.node())


def finalize_process(proc):
    """Wait for the process (clone, fetch, pull or push) and handle its errors accordingly"""
    proc.wait()

#} END utilities

#{ Classes


class RemoteProgress(object):

    """
    Handler providing an interface to parse progress information emitted by git-push
    and git-fetch and to dispatch callbacks allowing subclasses to react to the progress.
    """
    _num_op_codes = 7
    BEGIN, END, COUNTING, COMPRESSING, WRITING, RECEIVING, RESOLVING, FINDING_SOURCES = [1 << x for x in range(_num_op_codes)]
    STAGE_MASK = BEGIN | END
    OP_MASK = ~STAGE_MASK

    __slots__ = ("_cur_line", "_seen_ops")
    re_op_absolute = re.compile("(remote: )?([\w\s]+):\s+()(\d+)()(.*)")
    re_op_relative = re.compile("(remote: )?([\w\s]+):\s+(\d+)% \((\d+)/(\d+)\)(.*)")

    def __init__(self):
        self._seen_ops = list()

    def _parse_progress_line(self, line):
        """Parse progress information from the given line as retrieved by git-push
        or git-fetch

        :return: list(line, ...) list of lines that could not be processed"""<commit_msg>fix(util): Correct number of op codes<commit_after># utils.py
# Copyright (C) 2008, 2009 <NAME> (<EMAIL>) and contributors
#
# This module is part of GitPython and is released under
# the BSD License: http://www.opensource.org/licenses/bsd-license.php

import os
import re
import sys
import time
import stat
import shutil
import platform
import getpass
import threading

# NOTE:  Some of the unused imports might be used/imported by others.
# Handle once test-cases are back up and running.
from .exc import InvalidGitRepositoryError

from .compat import (
    MAXSIZE,
    defenc,
    PY3
)

# Most of these are unused here, but are for use by git-python modules so these
# don't see gitdb all the time. Flake of course doesn't like it.
from gitdb.util import (  # NOQA
    make_sha,
    LockedFD,
    file_contents_ro,
    LazyMixin,
    to_hex_sha,
    to_bin_sha
)

__all__ = ("stream_copy", "join_path", "to_native_path_windows", "to_native_path_linux",
           "join_path_native", "Stats", "IndexFileSHA1Writer", "Iterable", "IterableList",
           "BlockingLockFile", "LockFile", 'Actor', 'get_user_id', 'assure_directory_exists',
           'RemoteProgress', 'rmtree', 'WaitGroup', 'unbare_repo')

#{ Utility Methods


def unbare_repo(func):
    """Methods with this decorator raise InvalidGitRepositoryError if they
    encounter a bare repository"""

    def wrapper(self, *args, **kwargs):
        if self.repo.bare:
            raise InvalidGitRepositoryError("Method '%s' cannot operate on bare repositories" % func.__name__)
        # END bare method
        return func(self, *args, **kwargs)
    # END wrapper
    wrapper.__name__ = func.__name__
    return wrapper


def rmtree(path):
    """Remove the given recursively.

    :note: we use shutil rmtree but adjust its behaviour to see whether files that
        couldn't be deleted are read-only. Windows will not remove them in that case"""
    def onerror(func, path, exc_info):
        if not os.access(path, os.W_OK):
            # Is the error an access error ?
            os.chmod(path, stat.S_IWUSR)
            func(path)
        else:
            raise
    # END end onerror
    return shutil.rmtree(path, False, onerror)


def stream_copy(source, destination, chunk_size=512 * 1024):
    """Copy all data from the source stream into the destination stream in chunks
    of size chunk_size

    :return: amount of bytes written"""
    br = 0
    while True:
        chunk = source.read(chunk_size)
        destination.write(chunk)
        br += len(chunk)
        if len(chunk) < chunk_size:
            break
    # END reading output stream
    return br


def join_path(a, *p):
    """Join path tokens together similar to os.path.join, but always use
    '/' instead of possibly '\' on windows."""
    path = a
    for b in p:
        if len(b) == 0:
            continue
        if b.startswith('/'):
            path += b[1:]
        elif path == '' or path.endswith('/'):
            path += b
        else:
            path += '/' + b
    # END for each path token to add
    return path


if sys.platform.startswith('win'):
    def to_native_path_windows(path):
        return path.replace('/', '\\')

    def to_native_path_linux(path):
        return path.replace('\\', '/')

    to_native_path = to_native_path_windows
else:
    # no need for any work on linux
    def to_native_path_linux(path):
        return path
    to_native_path = to_native_path_linux


def join_path_native(a, *p):
    """
    As join path, but makes sure an OS native path is returned. This is only
        needed to play it safe on my dear windows and to assure nice paths that only
        use '\'"""
    return to_native_path(join_path(a, *p))


def assure_directory_exists(path, is_file=False):
    """Assure that the directory pointed to by path exists.

    :param is_file: If True, path is assumed to be a file and handled correctly.
        Otherwise it must be a directory
    :return: True if the directory was created, False if it already existed"""
    if is_file:
        path = os.path.dirname(path)
    # END handle file
    if not os.path.isdir(path):
        os.makedirs(path)
        return True
    return False


def get_user_id():
    """:return: string identifying the currently active system user as name@node"""
    return "%s@%s" % (getpass.getuser(), platform.node())


def finalize_process(proc):
    """Wait for the process (clone, fetch, pull or push) and handle its errors accordingly"""
    proc.wait()

#} END utilities

#{ Classes


class RemoteProgress(object):

    """
    Handler providing an interface to parse progress information emitted by git-push
    and git-fetch and to dispatch callbacks allowing subclasses to react to the progress.
    """
    _num_op_codes = 8
    BEGIN, END, COUNTING, COMPRESSING, WRITING, RECEIVING, RESOLVING, FINDING_SOURCES = [1 << x for x in range(_num_op_codes)]
    STAGE_MASK = BEGIN | END
    OP_MASK = ~STAGE_MASK

    __slots__ = ("_cur_line", "_seen_ops")
    re_op_absolute = re.compile("(remote: )?([\w\s]+):\s+()(\d+)()(.*)")
    re_op_relative = re.compile("(remote: )?([\w\s]+):\s+(\d+)% \((\d+)/(\d+)\)(.*)")

    def __init__(self):
        self._seen_ops = list()

    def _parse_progress_line(self, line):
        """Parse progress information from the given line as retrieved by git-push
        or git-fetch

        :return: list(line, ...) list of lines that could not be processed"""
{'fraction_non_alphanumeric': 0.10660412067696835, 'fraction_numerical': 0.004047093451066961, 'mean_word_length': 2.8721509971509973, 'pattern_counts': {'":': 2, '<': 13, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '854233', 'n_tokens_mistral': 3396, 'n_tokens_neox': 3260, 'n_words': 1148}