<commit_before>from datetime import datetime
import os
from stat import S_ISLNK, S_ISDIR, S_ISREG, S_IFDIR, S_IRWXU, S_IRWXG, S_IRWXO
import time

from dateutil.tz import tzlocal
from dulwich.objects import S_ISGITLINK, Blob, Commit, Tree
import logbook
from stuf.collects import ChainMap


log = logbook.Logger('git')


def export_tree(lookup, tree, path):
    """Exports the given tree object to path.

    :param lookup: Function to retrieve objects for SHA1 hashes.
    :param tree: Tree to export.
    :param path: Output path.
    """
    FILE_PERM = S_IRWXU | S_IRWXG | S_IRWXO

    for name, mode, hexsha in tree.iteritems():
        dest = os.path.join(path, name)

        if S_ISGITLINK(mode):
            log.error('Ignored submodule {}; submodules are not yet supported.'
                      .format(name))
            # raise ValueError('Does not support submodules')
        elif S_ISDIR(mode):
            os.mkdir(dest)
            os.chmod(dest, 0o0755)
            export_tree(lookup, lookup(hexsha), dest)
        elif S_ISLNK(mode):
            os.symlink(lookup(hexsha).data, dest)
        elif S_ISREG(mode):
            with open(dest, 'wb') as out:
                for chunk in lookup(hexsha).chunked:
                    out.write(chunk)
            os.chmod(dest, mode & FILE_PERM)
        else:
            raise ValueError('Cannot deal with mode of {:o} from {}'.format
                             (mode, name))


def get_local_timezone(now=None):
    if now is None:
        now = int(time.time())

    # calculate local timezone. must be done each time, due to DST
    _tz_offset = tzlocal().utcoffset(datetime.utcfromtimestamp(now))

    return _tz_offset.days * 24 * 60 * 60 + _tz_offset.seconds


def one_or_many(f):
    def _(self, *args, **kwargs):
        if len(self.candidates) == 1:
            return f(self, self.candidates[0], *args, **kwargs)
        if len(self.candidates) > 1:
            return [f(self, c) for c in self.candidates]
    return _


class ResolvedRef(object):
    """Resolves a commit-ish/tree-ish to an object.

    Correct way is described at `here <http://stackoverflow.com/questions/
    23303549/what-are-commit-ish-and-tree-ish-in-git>`_.

    Resolution in this function is much simpler, no ``@{}~:/^ `` are supported.
    """

    SYM_PREFIX = 'ref: '
    TAG_PREFIX = 'refs/tags/'

    def __init__(self, repo, ref, lookup=None):
        self.repo = repo
        self.ref = ref
        self.lookup = lookup or repo.object_store.__getitem__

        full_names = [
            ref,
            'refs/tags/{}'.format(ref),
            'refs/heads/{}'.format(ref),
            'refs/remotes/{}'.format(ref),
            'refs/{}'.format(ref),
        ]

        candidates = []

        if ref in self.repo.object_store:
            candidates.append((ref, 'object', None))

        for name in full_names:
            if name in self.repo.refs:
                # store symbolic ref targets
                target = self.repo.refs.read_ref(name)
                if not target.startswith(self.SYM_PREFIX):
                    target = None
                else:
                    target = target[len(self.SYM_PREFIX):]

                candidates.append((name, 'ref', target))

        self.candidates = candidates

    @property
    def is_definite(self):
        return len(self.candidates) < 2

    @property<commit_msg>Check hash is valid before rummaging in object store.<commit_after>from datetime import datetime
import os
import re
from stat import S_ISLNK, S_ISDIR, S_ISREG, S_IFDIR, S_IRWXU, S_IRWXG, S_IRWXO
import time

from dateutil.tz import tzlocal
from dulwich.objects import S_ISGITLINK, Blob, Commit, Tree
import logbook
from stuf.collects import ChainMap


log = logbook.Logger('git')
HASH_RE = re.compile('^[a-zA-Z0-9]{40}$')


def export_tree(lookup, tree, path):
    """Exports the given tree object to path.

    :param lookup: Function to retrieve objects for SHA1 hashes.
    :param tree: Tree to export.
    :param path: Output path.
    """
    FILE_PERM = S_IRWXU | S_IRWXG | S_IRWXO

    for name, mode, hexsha in tree.iteritems():
        dest = os.path.join(path, name)

        if S_ISGITLINK(mode):
            log.error('Ignored submodule {}; submodules are not yet supported.'
                      .format(name))
            # raise ValueError('Does not support submodules')
        elif S_ISDIR(mode):
            os.mkdir(dest)
            os.chmod(dest, 0o0755)
            export_tree(lookup, lookup(hexsha), dest)
        elif S_ISLNK(mode):
            os.symlink(lookup(hexsha).data, dest)
        elif S_ISREG(mode):
            with open(dest, 'wb') as out:
                for chunk in lookup(hexsha).chunked:
                    out.write(chunk)
            os.chmod(dest, mode & FILE_PERM)
        else:
            raise ValueError('Cannot deal with mode of {:o} from {}'.format
                             (mode, name))


def get_local_timezone(now=None):
    if now is None:
        now = int(time.time())

    # calculate local timezone. must be done each time, due to DST
    _tz_offset = tzlocal().utcoffset(datetime.utcfromtimestamp(now))

    return _tz_offset.days * 24 * 60 * 60 + _tz_offset.seconds


def one_or_many(f):
    def _(self, *args, **kwargs):
        if len(self.candidates) == 1:
            return f(self, self.candidates[0], *args, **kwargs)
        if len(self.candidates) > 1:
            return [f(self, c) for c in self.candidates]
    return _


class ResolvedRef(object):
    """Resolves a commit-ish/tree-ish to an object.

    Correct way is described at `here <http://stackoverflow.com/questions/
    23303549/what-are-commit-ish-and-tree-ish-in-git>`_.

    Resolution in this function is much simpler, no ``@{}~:/^ `` are supported.
    """

    SYM_PREFIX = 'ref: '
    TAG_PREFIX = 'refs/tags/'

    def __init__(self, repo, ref, lookup=None):
        self.repo = repo
        self.ref = ref
        self.lookup = lookup or repo.object_store.__getitem__

        candidates = []

        # only check sha1 hashes
        if HASH_RE.match(ref) and ref in self.repo.object_store:
            candidates.append((ref, 'object', None))

        full_names = [
            ref,
            'refs/tags/{}'.format(ref),
            'refs/heads/{}'.format(ref),
            'refs/remotes/{}'.format(ref),
            'refs/{}'.format(ref),
        ]

        for name in full_names:
            if name in self.repo.refs:
                # store symbolic ref targets
                target = self.repo.refs.read_ref(name)
                if not target.startswith(self.SYM_PREFIX):
                    target = None
                else:
                    target = target[len(self.SYM_PREFIX):]

                candidates.append((name, 'ref', target))

        self.candidates = candidates

    @property
    def is_definite(self):
        return len(self.candidates) < 2

    @property
{'fraction_non_alphanumeric': 0.11210374639769452, 'fraction_numerical': 0.007636887608069165, 'mean_word_length': 2.111160914388167, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1822074', 'n_tokens_mistral': 2273, 'n_tokens_neox': 2131, 'n_words': 569}