<commit_before>import datetime
import dulwich
import fnmatch
import os
import stat


class BaseObject(object):
    def __init__(self, odb, obj):
        self._odb = odb
        self._obj = obj
        self.sha = obj.sha().hexdigest()

        # will be setted by TreeObject or CommitObject
        self.parent = None
        self.name = None

        # will be setted through set_commit
        self._commit_sha = None

    @property
    def abs_name(self):
        paths = [self.name]
        parent = self.parent

        while parent is not None:
            paths.insert(0, parent.name)
            parent = parent.parent

        return os.path.join(*paths)

    @property
    def last_modified(self):
        visited = set()
        pendings = [self.commit]
        abs_name = self.abs_name
        while pendings:
            commit = pendings.pop(0)
            for parent_commit in commit.parents:
                try:
                    obj_sha = parent_commit.tree[abs_name].sha
                except KeyError:
                    obj_sha = None

                if self.sha == obj_sha:
                    # if this commit's file has same sha,
                    # the file was altered somewhere in this branch (correct?)
                    pendings.append(parent_commit)
                    break
            else:
                # all parents has different sha,
                # so this file was altered by this commit
                return commit.commit_time

        # maybe initial commit
        return commit.commit_time

    @property
    def commit(self):
        return self._odb.get_commit(self._commit_sha)

    def set_commit(self, commit_obj):
        if isinstance(commit_obj, CommitObject):
            # record hash to avoid circular reference
            self._commit_sha = commit_obj.sha

        elif isinstance(commit_obj, basestring):
            self._commit_sha = commit_obj

        else:
            raise ObjectTypeMismatch

    def __eq__(self, other):
        if hasattr(other, 'sha'):
            return self.sha == other.sha
        else:
            return False


class BlobObject(BaseObject):
    _size = None

    @property
    def data(self):
        return self._obj.data

    @property
    def size(self):
        if self._size is None:
            self._size = len(self.data)
        return self._size


class LinkObject(BlobObject):
    @property
    def target(self):
        return self.data


class TreeObject(BaseObject):
    is_tree = True

    def iteritems(self):
        for name, _, _ in self._obj.iteritems():
            yield (name, self[name])

    def __iter__(self):
        return iter(self._obj)

    def __contains__(self, name):
        if '/' in name:
            try:
                self[name]
                return True
            except KeyError:
                return False
        else:
            return name in self._obj

    def __getitem__(self, name):
        if '/' in name:
            return self._get_path(name)

        mode, hash = self._obj[name]

        if mode & stat.S_IFDIR:
            obj = self._odb.get_tree(hash)<commit_msg>handle "." and ".."<commit_after>import datetime
import dulwich
import fnmatch
import os
import stat


class BaseObject(object):
    def __init__(self, odb, obj):
        self._odb = odb
        self._obj = obj
        self.sha = obj.sha().hexdigest()

        # will be setted by TreeObject or CommitObject
        self.parent = None
        self.name = None

        # will be setted through set_commit
        self._commit_sha = None

    @property
    def abs_name(self):
        paths = [self.name]
        parent = self.parent

        while parent is not None:
            paths.insert(0, parent.name)
            parent = parent.parent

        return os.path.join(*paths)

    @property
    def last_modified(self):
        visited = set()
        pendings = [self.commit]
        abs_name = self.abs_name
        while pendings:
            commit = pendings.pop(0)
            for parent_commit in commit.parents:
                try:
                    obj_sha = parent_commit.tree[abs_name].sha
                except KeyError:
                    obj_sha = None

                if self.sha == obj_sha:
                    # if this commit's file has same sha,
                    # the file was altered somewhere in this branch (correct?)
                    pendings.append(parent_commit)
                    break
            else:
                # all parents has different sha,
                # so this file was altered by this commit
                return commit.commit_time

        # maybe initial commit
        return commit.commit_time

    @property
    def commit(self):
        return self._odb.get_commit(self._commit_sha)

    def set_commit(self, commit_obj):
        if isinstance(commit_obj, CommitObject):
            # record hash to avoid circular reference
            self._commit_sha = commit_obj.sha

        elif isinstance(commit_obj, basestring):
            self._commit_sha = commit_obj

        else:
            raise ObjectTypeMismatch

    def __eq__(self, other):
        if hasattr(other, 'sha'):
            return self.sha == other.sha
        else:
            return False


class BlobObject(BaseObject):
    _size = None

    @property
    def data(self):
        return self._obj.data

    @property
    def size(self):
        if self._size is None:
            self._size = len(self.data)
        return self._size


class LinkObject(BlobObject):
    @property
    def target(self):
        return self.data


class TreeObject(BaseObject):
    is_tree = True

    def iteritems(self):
        for name, _, _ in self._obj.iteritems():
            yield (name, self[name])

    def __iter__(self):
        return iter(self._obj)

    def __contains__(self, name):
        if '/' in name:
            try:
                self[name]
                return True
            except KeyError:
                return False
        else:
            return name in self._obj

    def __getitem__(self, name):
        if '/' in name:
            return self._get_path(name)

        if name == '.':
            return self

        elif name == '..':
            if self.parent:
                return self.parent
            else:
                # / (root)
                return self

        mode, hash = self._obj[name]

        if mode & stat.S_IFDIR:
            obj = self._odb.get_tree(hash)
{'fraction_non_alphanumeric': 0.07797811681306827, 'fraction_numerical': 0.0006164278008938204, 'mean_word_length': 1.556124458448208, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '715703', 'n_tokens_mistral': 1806, 'n_tokens_neox': 1701, 'n_words': 499}