<commit_before>"""
A hook into setuptools for Git.
"""
import sys
import os
import posixpath

from os.path import realpath, join
from subprocess import CalledProcessError
from subprocess import PIPE

from setuptools_git.utils import check_output
from setuptools_git.utils import b
from setuptools_git.utils import posix
from setuptools_git.utils import fsdecode
from setuptools_git.utils import hfs_quote
from setuptools_git.utils import compose
from setuptools_git.utils import decompose


def windecode(path):
    # We receive the raw bytes from Git and must decode by hand
    # (msysgit returns UTF-8 encoded bytes except when it doesn't)
    if sys.version_info >= (3,):
        try:
            path = path.decode('utf-8')
        except UnicodeDecodeError:
            path = path.decode(sys.getfilesystemencoding())
    else:
        try:
            path = path.decode('utf-8').encode(sys.getfilesystemencoding())
        except UnicodeError:
            pass
    return path


def gitlsfiles(dirname=''):
    # NB: Passing the '-z' option to 'git ls-files' below returns the
    # output as a blob of null-terminated filenames without canonical-
    # ization or use of double-quoting.
    #
    # So we'll get back e.g.:
    #
    # 'pyramid/tests/fixtures/static/h\xc3\xa9h\xc3\xa9.html'
    #
    # instead of:
    #
    # '"pyramid/tests/fixtures/static/h\\303\\251h\\303\\251.html"'
    #
    # for each file.
    res = set()

    try:
        topdir = check_output(
            ['git', 'rev-parse', '--show-toplevel'], cwd=dirname or None,
            stderr=PIPE).strip()

        if sys.platform == 'win32':
            cwd = windecode(topdir)
        else:
            cwd = topdir

        filenames = check_output(
            ['git', 'ls-files', '-z'], cwd=cwd, stderr=PIPE)
    except (CalledProcessError, OSError):
        # Setuptools mandates we fail silently
        return res

    for filename in filenames.split(b('\x00')):
        if filename:
            filename = posixpath.join(topdir, filename)
            if sys.platform == 'darwin':
                filename = hfs_quote(filename)
            if sys.platform == 'win32':
                filename = windecode(filename)
            else:
                filename = fsdecode(filename)
            if sys.platform == 'darwin':
                filename = decompose(filename)
            res.add(filename)
    return res


def listfiles(dirname=''):
    filenames = gitlsfiles(dirname)
    dirnames = set(posixpath.dirname(x) for x in filenames)

    cwd = realpath(dirname or os.curdir)
    prefix_length = len(cwd) + 1<commit_msg>Rename windecode -> ntfsdecode.<commit_after>"""
A hook into setuptools for Git.
"""
import sys
import os
import posixpath

from os.path import realpath, join
from subprocess import CalledProcessError
from subprocess import PIPE

from setuptools_git.utils import check_output
from setuptools_git.utils import b
from setuptools_git.utils import posix
from setuptools_git.utils import fsdecode
from setuptools_git.utils import hfs_quote
from setuptools_git.utils import compose
from setuptools_git.utils import decompose


def ntfsdecode(path):
    # We receive the raw bytes from Git and must decode by hand
    if sys.version_info >= (3,):
        try:
            path = path.decode('utf-8')
        except UnicodeDecodeError:
            path = path.decode(sys.getfilesystemencoding())
    else:
        try:
            path = path.decode('utf-8').encode(sys.getfilesystemencoding())
        except UnicodeError:
            pass  # Already in filesystem encoding (hopefully)
    return path


def gitlsfiles(dirname=''):
    # NB: Passing the '-z' option to 'git ls-files' below returns the
    # output as a blob of null-terminated filenames without canonical-
    # ization or use of double-quoting.
    #
    # So we'll get back e.g.:
    #
    # 'pyramid/tests/fixtures/static/h\xc3\xa9h\xc3\xa9.html'
    #
    # instead of:
    #
    # '"pyramid/tests/fixtures/static/h\\303\\251h\\303\\251.html"'
    #
    # for each file.
    res = set()

    try:
        topdir = check_output(
            ['git', 'rev-parse', '--show-toplevel'], cwd=dirname or None,
            stderr=PIPE).strip()

        if sys.platform == 'win32':
            cwd = ntfsdecode(topdir)
        else:
            cwd = topdir

        filenames = check_output(
            ['git', 'ls-files', '-z'], cwd=cwd, stderr=PIPE)
    except (CalledProcessError, OSError):
        # Setuptools mandates we fail silently
        return res

    for filename in filenames.split(b('\x00')):
        if filename:
            filename = posixpath.join(topdir, filename)
            if sys.platform == 'darwin':
                filename = hfs_quote(filename)
            if sys.platform == 'win32':
                filename = ntfsdecode(filename)
            else:
                filename = fsdecode(filename)
            if sys.platform == 'darwin':
                filename = decompose(filename)
            res.add(filename)
    return res


def listfiles(dirname=''):
    filenames = gitlsfiles(dirname)
    dirnames = set(posixpath.dirname(x) for x in filenames)

    cwd = realpath(dirname or os.curdir)
    prefix_length = len(cwd) + 1
{'fraction_non_alphanumeric': 0.10195103289977046, 'fraction_numerical': 0.010137719969395562, 'mean_word_length': 2.364864864864865, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1522796', 'n_tokens_mistral': 1653, 'n_tokens_neox': 1575, 'n_words': 435}