<commit_before># -*- coding: utf-8 -*-
"""Git Repo object for libvcs.

libvcs.git
~~~~~~~~~~

From https://github.com/saltstack/salt (Apache License):

- :py:meth:`GitRepo.remote`
- :py:meth:`GitRepo.remote_get`
- :py:meth:`GitRepo.remote_set`

From pip (MIT Licnese):

- :py:meth:`GitRepo.get_url_and_revision_from_pip_url` (get_url_rev)
- :py:meth:`GitRepo.get_revision`

"""
from __future__ import absolute_import, print_function, unicode_literals

import logging
import os
import re

from . import exc
from ._compat import urlparse
from .base import BaseRepo

logger = logging.getLogger(__name__)


class GitRepo(BaseRepo):
    bin_name = 'git'
    schemes = ('git', 'git+http', 'git+https', 'git+ssh', 'git+git', 'git+file')

    def __init__(self, url, remotes=None, **kwargs):
        """A git repository.

        :param url: URL of repo
        :type url: str

        :param remotes: list of remotes in dict format::

            [{
            "remote_name": "myremote",
            "url": "https://github.com/tony/vim-config.git"
            }]
        :type remotes: list

        :param git_remote_name: name of the remote (default "origin")
        :type git_remote_name: str

        :param git_shallow: clone with ``--depth 1`` (default False)
        :type git_shallow: bool

        :param git_submodules: Git submodules that shall be updated, all if
            empty
        :type git_submodules: list

        :param tls_verify: Should certificate for https be checked (default
            False)
        :type tls_verify: bool
        """
        if 'git_remote_name' not in kwargs:
            self.git_remote_name = "origin"
        if 'git_shallow' not in kwargs:
            self.git_shallow = False
        if 'git_submodules' not in kwargs:
            self.git_submodules = []
        if 'tls_verify' not in kwargs:
            self.tls_verify = False
        BaseRepo.__init__(self, url, **kwargs)

        self.remotes = remotes

    def get_revision(self):
        """Return current revision. Initial repositories return 'initial'."""
        try:
            return self.run(['rev-parse', '--verify', 'HEAD'])
        except exc.CommandError:
            return 'initial'

    @classmethod
    def get_url_and_revision_from_pip_url(cls, pip_url):
        """
        Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'.
        That's required because although they use SSH they sometimes doesn't
        work with a ssh:// scheme (e.g. Github). But we need a scheme for
        parsing. Hence we remove it again afterwards and return it as a stub.
        The manpage for git-clone(1) refers to this as the "scp-like styntax".
        """
        if '://' not in pip_url:
            assert 'file:' not in pip_url
            pip_url = pip_url.replace('git+', 'git+ssh://')
            url, rev = super(GitRepo, cls).get_url_and_revision_from_pip_url(pip_url)
            url = url.replace('ssh://', '')
        elif 'github.com:' in pip_url:
            raise exc.LibVCSException(
                "Repo %s is malformatted, please use the convention %s for"
                "ssh / private GitHub repositories."
                % (pip_url, "git+https://github.com/username/repo.git")
            )
        else:
            url, rev = super(GitRepo, cls).get_url_and_revision_from_pip_url(pip_url)

        return url, rev

    def obtain(self):
        """Retrieve the repository, clone if doesn't exist."""
        self.check_destination()

        url = self.url

        cmd = ['clone', '--progress']
        if self.git_shallow:
            cmd.extend(['--depth', '1'])
        if self.tls_verify:
            cmd.extend(['-c', 'http.sslVerify=false'])
        cmd.extend([url, self.path])

        self.info('Cloning.')
        self.run(cmd, log_in_real_time=True)

        if self.remotes:
            for r in self.remotes:
                self.error('Adding remote %s <%s>' % (r['remote_name'], r['url']))
                self.remote_set(name=r['remote_name'], url=r['url'])

        self.info('Initializing submodules.')
        self.run(['submodule', 'init'], log_in_real_time=True)
        cmd = ['submodule', 'update', '--recursive', '--init']
        cmd.extend(self.git_submodules)
        self.run(cmd, log_in_real_time=True)

    def update_repo(self):
        self.check_destination()

        if not os.path.isdir(os.path.join(self.path, '.git')):
            self.obtain()
            self.update_repo()
            return

        # Get requested revision or tag
        url, git_tag = self.url, getattr(self, 'rev', None)

        if not git_tag:
            self.debug("No git revision set, defaulting to origin/master")
            symref = self.run(['symbolic-ref', '--short', 'HEAD'])
            if symref:
                git_tag = symref.rstrip()
            else:
                git_tag = 'origin/master'
        self.debug("git_tag: %s" % git_tag)

        self.info("Updating to '%s'." % git_tag)

        # Get head sha<commit_msg>obtain: Overwrite URL if exists<commit_after># -*- coding: utf-8 -*-
"""Git Repo object for libvcs.

libvcs.git
~~~~~~~~~~

From https://github.com/saltstack/salt (Apache License):

- :py:meth:`GitRepo.remote`
- :py:meth:`GitRepo.remote_get`
- :py:meth:`GitRepo.remote_set`

From pip (MIT Licnese):

- :py:meth:`GitRepo.get_url_and_revision_from_pip_url` (get_url_rev)
- :py:meth:`GitRepo.get_revision`

"""
from __future__ import absolute_import, print_function, unicode_literals

import logging
import os
import re

from . import exc
from ._compat import urlparse
from .base import BaseRepo

logger = logging.getLogger(__name__)


class GitRepo(BaseRepo):
    bin_name = 'git'
    schemes = ('git', 'git+http', 'git+https', 'git+ssh', 'git+git', 'git+file')

    def __init__(self, url, remotes=None, **kwargs):
        """A git repository.

        :param url: URL of repo
        :type url: str

        :param remotes: list of remotes in dict format::

            [{
            "remote_name": "myremote",
            "url": "https://github.com/tony/vim-config.git"
            }]
        :type remotes: list

        :param git_remote_name: name of the remote (default "origin")
        :type git_remote_name: str

        :param git_shallow: clone with ``--depth 1`` (default False)
        :type git_shallow: bool

        :param git_submodules: Git submodules that shall be updated, all if
            empty
        :type git_submodules: list

        :param tls_verify: Should certificate for https be checked (default
            False)
        :type tls_verify: bool
        """
        if 'git_remote_name' not in kwargs:
            self.git_remote_name = "origin"
        if 'git_shallow' not in kwargs:
            self.git_shallow = False
        if 'git_submodules' not in kwargs:
            self.git_submodules = []
        if 'tls_verify' not in kwargs:
            self.tls_verify = False
        BaseRepo.__init__(self, url, **kwargs)

        self.remotes = remotes

    def get_revision(self):
        """Return current revision. Initial repositories return 'initial'."""
        try:
            return self.run(['rev-parse', '--verify', 'HEAD'])
        except exc.CommandError:
            return 'initial'

    @classmethod
    def get_url_and_revision_from_pip_url(cls, pip_url):
        """
        Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'.
        That's required because although they use SSH they sometimes doesn't
        work with a ssh:// scheme (e.g. Github). But we need a scheme for
        parsing. Hence we remove it again afterwards and return it as a stub.
        The manpage for git-clone(1) refers to this as the "scp-like styntax".
        """
        if '://' not in pip_url:
            assert 'file:' not in pip_url
            pip_url = pip_url.replace('git+', 'git+ssh://')
            url, rev = super(GitRepo, cls).get_url_and_revision_from_pip_url(pip_url)
            url = url.replace('ssh://', '')
        elif 'github.com:' in pip_url:
            raise exc.LibVCSException(
                "Repo %s is malformatted, please use the convention %s for"
                "ssh / private GitHub repositories."
                % (pip_url, "git+https://github.com/username/repo.git")
            )
        else:
            url, rev = super(GitRepo, cls).get_url_and_revision_from_pip_url(pip_url)

        return url, rev

    def obtain(self):
        """Retrieve the repository, clone if doesn't exist."""
        self.check_destination()

        url = self.url

        cmd = ['clone', '--progress']
        if self.git_shallow:
            cmd.extend(['--depth', '1'])
        if self.tls_verify:
            cmd.extend(['-c', 'http.sslVerify=false'])
        cmd.extend([url, self.path])

        self.info('Cloning.')
        self.run(cmd, log_in_real_time=True)

        if self.remotes:
            for r in self.remotes:
                self.error('Adding remote %s <%s>' % (r['remote_name'], r['url']))
                self.remote_set(name=r['remote_name'], url=r['url'], overwrite=True)

        self.info('Initializing submodules.')
        self.run(['submodule', 'init'], log_in_real_time=True)
        cmd = ['submodule', 'update', '--recursive', '--init']
        cmd.extend(self.git_submodules)
        self.run(cmd, log_in_real_time=True)

    def update_repo(self):
        self.check_destination()

        if not os.path.isdir(os.path.join(self.path, '.git')):
            self.obtain()
            self.update_repo()
            return

        # Get requested revision or tag
        url, git_tag = self.url, getattr(self, 'rev', None)

        if not git_tag:
            self.debug("No git revision set, defaulting to origin/master")
            symref = self.run(['symbolic-ref', '--short', 'HEAD'])
            if symref:
                git_tag = symref.rstrip()
            else:
                git_tag = 'origin/master'
        self.debug("git_tag: %s" % git_tag)

        self.info("Updating to '%s'." % git_tag)

        # Get head sha
{'fraction_non_alphanumeric': 0.12768282988871224, 'fraction_numerical': 0.000794912559618442, 'mean_word_length': 2.207456978967495, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1299262', 'n_tokens_mistral': 3195, 'n_tokens_neox': 3083, 'n_words': 808}