<commit_before># -*- coding: utf-8 -*-
"""
    vcs.backends.git
    ~~~~~~~~~~~~~~~~

    Git backend implementation.

    :created_on: Apr 8, 2010
    :copyright: (c) 2010-2011 by <NAME>, <NAME>.
"""

import os
import re
import time
import datetime
import posixpath
from dulwich import objects
from dulwich.repo import Repo, NotGitRepository
from itertools import chain
from string import Template
from subprocess import Popen, PIPE
from vcs.backends import ARCHIVE_SPECS
from vcs.backends.base import BaseChangeset
from vcs.backends.base import BaseInMemoryChangeset
from vcs.backends.base import BaseRepository
from vcs.backends.base import BaseWorkdir
from vcs.exceptions import BranchDoesNotExistError
from vcs.exceptions import ChangesetDoesNotExistError
from vcs.exceptions import ChangesetError
from vcs.exceptions import EmptyRepositoryError
from vcs.exceptions import ImproperArchiveTypeError
from vcs.exceptions import NodeDoesNotExistError
from vcs.exceptions import RepositoryError
from vcs.exceptions import TagAlreadyExistError
from vcs.exceptions import TagDoesNotExistError
from vcs.exceptions import VCSError
from vcs.nodes import FileNode, DirNode, NodeKind, RootNode, RemovedFileNode
from vcs.utils import safe_unicode, makedate, date_fromtimestamp
from vcs.utils.lazy import LazyProperty
from vcs.utils.ordered_dict import OrderedDict
from vcs.utils.paths import abspath


class GitRepository(BaseRepository):
    """
    Git repository backend.
    """
    DEFAULT_BRANCH_NAME = 'master'

    def __init__(self, repo_path, create=False, src_url=None,
                 update_after_clone=False):

        self.path = abspath(repo_path)
        self._repo = self._get_repo(create, src_url, update_after_clone)
        try:
            self.head = self._repo.head()
        except KeyError:
            self.head = None

    @LazyProperty
    def revisions(self):
        """
        Returns list of revisions' ids, in ascending order.  Being lazy
        attribute allows external tools to inject shas from cache.
        """
        return self._get_all_revisions()

    def run_git_command(self, cmd):
        """
        Runs given ``cmd`` as git command and returns tuple
        (returncode, stdout, stderr).

        .. note::
           This method exists only until log/blame functionality is implemented
           at Dulwich (see https://bugs.launchpad.net/bugs/645142). Parsing
           os command's output is road to hell...

        :param cmd: git command to be executed
        """
        #cmd = '(cd %s && git %s)' % (self.path, cmd)
        cmd = 'git %s' % cmd
        try:
            opts = dict(
                shell=True,
                stdout=PIPE,
                stderr=PIPE)
            if os.path.isdir(self.path):
                opts['cwd'] = self.path
            p = Popen(cmd, **opts)
        except OSError, err:
            raise RepositoryError("Couldn't run git command (%s).\n"
                "Original error was:%s" % (cmd, err))
        so, se = p.communicate()
        if not se.startswith("fatal: bad default revision 'HEAD'") and \
            p.returncode != 0:
            raise RepositoryError("Couldn't run git command (%s).\n"
                "stderr:\n%s" % (cmd, se))
        return so, se

    def _get_diff(self, rev1, rev2, path):
        cmd = 'diff %s %s %s' % (rev1, rev2, path)
        so, se = self.run_git_command(cmd)

        return so

    def _check_url(self, url):
        """
        Functon will check given url and try to verify if it's a valid
        link. Sometimes it may happened that mercurial will issue basic
        auth request that can cause whole API to hang when used from python
        or other external calls.
        
        On failures it'll raise urllib2.HTTPError
        """

        #TODO: implement this
        pass


    def _get_repo(self, create, src_url=None, update_after_clone=False):
        if create and os.path.exists(self.path):
            raise RepositoryError("Location already exist")
        if src_url and not create:
            raise RepositoryError("Create should be set to True if src_url is "
                                  "given (clone operation creates repository)")
        try:
            if create and src_url:
                self._check_url(src_url)
                self.clone(src_url, update_after_clone)
                return Repo(self.path)
            elif create:
                os.mkdir(self.path)
                return Repo.init(self.path)
            else:
                return Repo(self.path)
        except (NotGitRepository, OSError), err:
            raise RepositoryError(err)

    def _get_all_revisions(self):
        cmd = 'rev-list --all --date-order'
        try:
            so, se = self.run_git_command(cmd)
        except RepositoryError:
            # Can be raised for empty repositories
            return []
        revisions = so.splitlines()
        revisions.reverse()
        return revisions

    def _get_revision(self, revision):
        """
        For git backend we always return integer here. This way we ensure
        that changset's revision attribute would become integer.
        """
        pattern = re.compile(r'^[[0-9a-fA-F]{12}|[0-9a-fA-F]{40}]$')

        if len(self.revisions) == 0:
            raise EmptyRepositoryError("There are no changesets yet")

        if revision in (None, 'tip', 'HEAD', 'head', -1):
            revision = self.revisions[-1]

        if (isinstance(revision, (str, unicode)) and revision.isdigit() \
            and len(revision) < 12) or isinstance(revision, int):
            try:
                revision = self.revisions[int(revision)]
            except:
                raise ChangesetDoesNotExistError("Revision %r does not exist "
                    "for this repository %s" % (revision, self))

        elif isinstance(revision, (str, unicode)):
            if not pattern.match(revision):
                raise ChangesetDoesNotExistError("Revision %r does not exist "
                    "for this repository %s" % (revision, self))
            try:
                revision = self.revisions[self.revisions.index(revision)]
            except ValueError:
                raise ChangesetDoesNotExistError("Revision %r does not exist "
                    "for this repository %s" % (revision, self))

        # Ensure we return full id
        if not pattern.match(str(revision)):
            raise ChangesetDoesNotExistError("Given revision %r not recognized"
                % revision)
        return revision

    def _get_archives(self, archive_name='tip'):

        for i in [('zip', '.zip'), ('gz', '.tar.gz'), ('bz2', '.tar.bz2')]:
                yield {"type": i[0], "extension": i[1], "node": archive_name}

    def _get_url(self, url):
        """
        Returns normalized url. If schema is not given, would fall to
        filesystem (``file://``) schema.
        """<commit_msg>Simplified some checks in GitRepository._get_revision.<commit_after># -*- coding: utf-8 -*-
"""
    vcs.backends.git
    ~~~~~~~~~~~~~~~~

    Git backend implementation.

    :created_on: Apr 8, 2010
    :copyright: (c) 2010-2011 by <NAME>, <NAME>.
"""

import os
import re
import time
import datetime
import posixpath
from dulwich import objects
from dulwich.repo import Repo, NotGitRepository
from itertools import chain
from string import Template
from subprocess import Popen, PIPE
from vcs.backends import ARCHIVE_SPECS
from vcs.backends.base import BaseChangeset
from vcs.backends.base import BaseInMemoryChangeset
from vcs.backends.base import BaseRepository
from vcs.backends.base import BaseWorkdir
from vcs.exceptions import BranchDoesNotExistError
from vcs.exceptions import ChangesetDoesNotExistError
from vcs.exceptions import ChangesetError
from vcs.exceptions import EmptyRepositoryError
from vcs.exceptions import ImproperArchiveTypeError
from vcs.exceptions import NodeDoesNotExistError
from vcs.exceptions import RepositoryError
from vcs.exceptions import TagAlreadyExistError
from vcs.exceptions import TagDoesNotExistError
from vcs.exceptions import VCSError
from vcs.nodes import FileNode, DirNode, NodeKind, RootNode, RemovedFileNode
from vcs.utils import safe_unicode, makedate, date_fromtimestamp
from vcs.utils.lazy import LazyProperty
from vcs.utils.ordered_dict import OrderedDict
from vcs.utils.paths import abspath


class GitRepository(BaseRepository):
    """
    Git repository backend.
    """
    DEFAULT_BRANCH_NAME = 'master'

    def __init__(self, repo_path, create=False, src_url=None,
                 update_after_clone=False):

        self.path = abspath(repo_path)
        self._repo = self._get_repo(create, src_url, update_after_clone)
        try:
            self.head = self._repo.head()
        except KeyError:
            self.head = None

    @LazyProperty
    def revisions(self):
        """
        Returns list of revisions' ids, in ascending order.  Being lazy
        attribute allows external tools to inject shas from cache.
        """
        return self._get_all_revisions()

    def run_git_command(self, cmd):
        """
        Runs given ``cmd`` as git command and returns tuple
        (returncode, stdout, stderr).

        .. note::
           This method exists only until log/blame functionality is implemented
           at Dulwich (see https://bugs.launchpad.net/bugs/645142). Parsing
           os command's output is road to hell...

        :param cmd: git command to be executed
        """
        #cmd = '(cd %s && git %s)' % (self.path, cmd)
        cmd = 'git %s' % cmd
        try:
            opts = dict(
                shell=True,
                stdout=PIPE,
                stderr=PIPE)
            if os.path.isdir(self.path):
                opts['cwd'] = self.path
            p = Popen(cmd, **opts)
        except OSError, err:
            raise RepositoryError("Couldn't run git command (%s).\n"
                "Original error was:%s" % (cmd, err))
        so, se = p.communicate()
        if not se.startswith("fatal: bad default revision 'HEAD'") and \
            p.returncode != 0:
            raise RepositoryError("Couldn't run git command (%s).\n"
                "stderr:\n%s" % (cmd, se))
        return so, se

    def _get_diff(self, rev1, rev2, path):
        cmd = 'diff %s %s %s' % (rev1, rev2, path)
        so, se = self.run_git_command(cmd)

        return so

    def _check_url(self, url):
        """
        Functon will check given url and try to verify if it's a valid
        link. Sometimes it may happened that mercurial will issue basic
        auth request that can cause whole API to hang when used from python
        or other external calls.

        On failures it'll raise urllib2.HTTPError
        """

        #TODO: implement this
        pass


    def _get_repo(self, create, src_url=None, update_after_clone=False):
        if create and os.path.exists(self.path):
            raise RepositoryError("Location already exist")
        if src_url and not create:
            raise RepositoryError("Create should be set to True if src_url is "
                                  "given (clone operation creates repository)")
        try:
            if create and src_url:
                self._check_url(src_url)
                self.clone(src_url, update_after_clone)
                return Repo(self.path)
            elif create:
                os.mkdir(self.path)
                return Repo.init(self.path)
            else:
                return Repo(self.path)
        except (NotGitRepository, OSError), err:
            raise RepositoryError(err)

    def _get_all_revisions(self):
        cmd = 'rev-list --all --date-order'
        try:
            so, se = self.run_git_command(cmd)
        except RepositoryError:
            # Can be raised for empty repositories
            return []
        revisions = so.splitlines()
        revisions.reverse()
        return revisions

    def _get_revision(self, revision):
        """
        For git backend we always return integer here. This way we ensure
        that changset's revision attribute would become integer.
        """
        pattern = re.compile(r'^[[0-9a-fA-F]{12}|[0-9a-fA-F]{40}]$')

        if len(self.revisions) == 0:
            raise EmptyRepositoryError("There are no changesets yet")

        if revision in (None, 'tip', 'HEAD', 'head', -1):
            revision = self.revisions[-1]

        if (isinstance(revision, (str, unicode)) and revision.isdigit() \
            and len(revision) < 12) or isinstance(revision, int):
            try:
                revision = self.revisions[int(revision)]
            except:
                raise ChangesetDoesNotExistError("Revision %r does not exist "
                    "for this repository %s" % (revision, self))

        elif isinstance(revision, (str, unicode)):
            if not pattern.match(revision) or revision not in self.revisions:
                raise ChangesetDoesNotExistError("Revision %r does not exist "
                    "for this repository %s" % (revision, self))

        # Ensure we return full id
        if not pattern.match(str(revision)):
            raise ChangesetDoesNotExistError("Given revision %r not recognized"
                % revision)
        return revision

    def _get_archives(self, archive_name='tip'):

        for i in [('zip', '.zip'), ('gz', '.tar.gz'), ('bz2', '.tar.bz2')]:
                yield {"type": i[0], "extension": i[1], "node": archive_name}

    def _get_url(self, url):
        """
        Returns normalized url. If schema is not given, would fall to
        filesystem (``file://``) schema.
        """
{'fraction_non_alphanumeric': 0.09525550113312377, 'fraction_numerical': 0.006287009284304408, 'mean_word_length': 2.2401705352913313, 'pattern_counts': {'":': 6, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '71621', 'n_tokens_mistral': 4026, 'n_tokens_neox': 3787, 'n_words': 1198}