<commit_before>#! /usr/bin/env python
"""Git-related utility functions."""

import os
import re

from .utils import which, check_output, system, cd, status


_SHA1_REGEX = r'^[0-9a-f]{40}$'
_SHA1_PATTERN = re.compile(_SHA1_REGEX)


def is_sha1(string):
    """Test if a string is a SHA1.

    Parameters
    ----------
    string : str
        String to test.

    Returns
    -------
    boolean
        ``True`` if the string is a valid SHA1. Otherwise, ``False``.
    """
    return _SHA1_PATTERN.match(string) is not None


def git_repo_name(url):
    """Get the name of a git repository.

    Parameters
    ----------
    url : str
        URL of a git repository.

    Returns
    -------
    str
        Name of the repository.
    """
    (base, _) = os.path.splitext(os.path.basename(url))
    return base


def git_repo_sha(url, git=None, branch='master'):
    """Get the SHA for a git repository.

    Parameters
    ----------
    url : str
        URL of a git repository.
    git : str, optional
        Path to the git program.
    branch : str, optional
        Branch of the git repository.

    Returns
    -------
    str
        First 10 characters of the SHA.
    """
    git = git or which('git')

    lines = check_output([git, 'ls-remote', url]).strip().split(os.linesep)
    shas = dict()
    for line in lines:
        (sha, name) = line.split()
        shas[name] = sha

    return shas['refs/heads/{branch}'.format(branch=branch)][:10]


def git_clone(url, git=None, dir='.', branch='master'):
    """Clone a git repository.

    Parameters
    ----------
    url : str
        URL of a git repository.
    git : str, optional
        Path to the git program.
    dir : str, optional
        Path to a folder to clone into.
    branch : str, optional
        Branch of the git repository.
    """
    git = git or which('git')

    with cd(dir):
        system([git, 'init', '-q'])
        system([git, 'config', 'remote.origin.url', url])
        system([git, 'config', 'remote.origin.fetch',
                '+refs/heads/*:refs/remotes/origin/*'])
        system([git, 'fetch', 'origin',
                '{branch}:refs/remotes/origin/{branch}'.format(branch=branch),
                '-n', '--depth=1'])
        system([git, 'reset', '--hard',
                'origin/{branch}'.format(branch=branch)])


def git_pull(dir='.', branch='master', git=None):
    """Pull from a git repository.

    Parameters
    ----------
    git : str, optional
        Path to the git program.
    dir : str, optional
        Path to a folder that contains the git repository.
    branch : str, optional
        Branch of the git repository.
    """
    git = git or which('git')

    with cd(dir):
        system(['git', 'checkout', '-q', branch])
        system(['git', 'pull', 'origin', '-q',
                ':'.join([
                    'refs/heads/{branch}',
                    'refs/remotes/origin/{branch}']).format(branch=branch)])


def git_clone_or_update(url, dir='.', branch='master', git=None):
    """Clone (or update) a git repository .

    If the repository doesn't not exists locally, clone it from *url*.
    Otherwise, simply update it.

    Parameters
    ----------
    url : str
        URL of a git repository.
    git : str, optional
        Path to the git program.
    dir : str, optional
        Path to a folder that contains the git repository.
    branch : str, optional
        Branch of the git repository.
    """
    if os.path.isdir(os.path.join(dir, '.git')):
        status('Updating %s' % url)
        git_pull(dir=dir, branch=branch, git=git)
    else:
        status('Cloning %s' % url)
        git_clone(url, dir=dir, branch=branch, git=git)
<commit_msg>Added git checkout when cloning.<commit_after>#! /usr/bin/env python
"""Git-related utility functions."""

import os
import re

from .utils import which, check_output, system, cd, status


_SHA1_REGEX = r'^[0-9a-f]{40}$'
_SHA1_PATTERN = re.compile(_SHA1_REGEX)


def is_sha1(string):
    """Test if a string is a SHA1.

    Parameters
    ----------
    string : str
        String to test.

    Returns
    -------
    boolean
        ``True`` if the string is a valid SHA1. Otherwise, ``False``.
    """
    return _SHA1_PATTERN.match(string) is not None


def git_repo_name(url):
    """Get the name of a git repository.

    Parameters
    ----------
    url : str
        URL of a git repository.

    Returns
    -------
    str
        Name of the repository.
    """
    (base, _) = os.path.splitext(os.path.basename(url))
    return base


def git_repo_sha(url, git=None, branch='master'):
    """Get the SHA for a git repository.

    Parameters
    ----------
    url : str
        URL of a git repository.
    git : str, optional
        Path to the git program.
    branch : str, optional
        Branch of the git repository.

    Returns
    -------
    str
        First 10 characters of the SHA.
    """
    git = git or which('git')

    lines = check_output([git, 'ls-remote', url]).strip().split(os.linesep)
    shas = dict()
    for line in lines:
        (sha, name) = line.split()
        shas[name] = sha

    return shas['refs/heads/{branch}'.format(branch=branch)][:10]


def git_clone(url, git=None, dir='.', branch='master'):
    """Clone a git repository.

    Parameters
    ----------
    url : str
        URL of a git repository.
    git : str, optional
        Path to the git program.
    dir : str, optional
        Path to a folder to clone into.
    branch : str, optional
        Branch of the git repository.
    """
    git = git or which('git')

    with cd(dir):
        system([git, 'init', '-q'])
        system([git, 'config', 'remote.origin.url', url])
        system([git, 'config', 'remote.origin.fetch',
                '+refs/heads/*:refs/remotes/origin/*'])
        system([git, 'fetch', 'origin',
                '{branch}:refs/remotes/origin/{branch}'.format(branch=branch),
                '-n', '--depth=1'])
        system([git, 'reset', '--hard',
                'origin/{branch}'.format(branch=branch)])
        system([git, 'checkout', '-q', branch])


def git_pull(dir='.', branch='master', git=None):
    """Pull from a git repository.

    Parameters
    ----------
    git : str, optional
        Path to the git program.
    dir : str, optional
        Path to a folder that contains the git repository.
    branch : str, optional
        Branch of the git repository.
    """
    git = git or which('git')

    with cd(dir):
        system(['git', 'checkout', '-q', branch])
        system(['git', 'pull', 'origin', '-q',
                ':'.join([
                    'refs/heads/{branch}',
                    'refs/remotes/origin/{branch}']).format(branch=branch)])


def git_clone_or_update(url, dir='.', branch='master', git=None):
    """Clone (or update) a git repository .

    If the repository doesn't not exists locally, clone it from *url*.
    Otherwise, simply update it.

    Parameters
    ----------
    url : str
        URL of a git repository.
    git : str, optional
        Path to the git program.
    dir : str, optional
        Path to a folder that contains the git repository.
    branch : str, optional
        Branch of the git repository.
    """
    if os.path.isdir(os.path.join(dir, '.git')):
        status('Updating %s' % url)
        git_pull(dir=dir, branch=branch, git=git)
    else:
        status('Cloning %s' % url)
        git_clone(url, dir=dir, branch=branch, git=git)

{'fraction_non_alphanumeric': 0.1525830997196636, 'fraction_numerical': 0.004271792818048324, 'mean_word_length': 2.02462656439241, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '754806', 'n_tokens_mistral': 2308, 'n_tokens_neox': 2227, 'n_words': 706}