<commit_before>import os
import re
import subprocess

from django.utils.text import slugify
from django.conf import settings
from django.core.cache import cache

# These options are passed to Fabric as: fab task --abort-on-prompts=True --user=root ...
fabric_special_options = ['no_agent', 'forward-agent', 'config', 'disable-known-hosts', 'keepalive',
                          'password', 'parallel', 'no-pty', 'reject-unknown-hosts', 'skip-bad-hosts', 'timeout',
                          'command-timeout', 'user', 'warn-only', 'pool-size']


def check_output(command, shell=False):
    return subprocess.check_output(command, shell=shell, executable=getattr(settings, 'SHELL', '/bin/sh'))


def check_output_with_ssh_key(command):
    if getattr(settings, 'GIT_SSH_KEY_LOCATION', None):
        return check_output('ssh-agent bash -c "ssh-add {};{}"'.format(settings.GIT_SSH_KEY_LOCATION, command),
                            shell=True)
    else:
        return check_output([command], shell=True)


def update_project_git(project, cache_dir, repo_dir):
    if not os.path.exists(repo_dir):
        if not os.path.exists(cache_dir):
            os.makedirs(cache_dir)

        check_output_with_ssh_key('git clone {} {}'.format(project.repo_url, repo_dir))
    else:
        check_output_with_ssh_key(
            'cd {0};git stash;git pull'.format(repo_dir)
        )


def setup_virtual_env_if_needed(repo_dir):
    env_dir = os.path.join(repo_dir, 'env')
    if not os.path.exists(env_dir):
        os.makedirs(env_dir)
        check_output("virtualenv {}".format(env_dir))


def update_project_requirements(project, repo_dir, activate_loc):
    pip_installs = ' '.join(project.fabfile_requirements.splitlines())

    check_output_with_ssh_key('source {} && cd {};pip install {}'.format(activate_loc, repo_dir, pip_installs))


def get_fabfile_path(project):
    if project.use_repo_fabfile:
        cache_key = 'project_{}_fabfile_path'.format(project.pk)
        cached_result = cache.get(cache_key)

        if cached_result:
            return cached_result

        cache_dir = os.path.join(settings.PUBLIC_DIR, '.repo_caches')
        repo_dir = os.path.join(cache_dir, slugify(project.name))

        update_project_git(project, cache_dir, repo_dir)
        setup_virtual_env_if_needed(repo_dir)
        activate_loc = os.path.join(repo_dir, 'env', 'bin', 'activate')
<commit_msg>Virtualenv creation command needs a shell<commit_after>import os
import re
import subprocess

from django.utils.text import slugify
from django.conf import settings
from django.core.cache import cache

# These options are passed to Fabric as: fab task --abort-on-prompts=True --user=root ...
fabric_special_options = ['no_agent', 'forward-agent', 'config', 'disable-known-hosts', 'keepalive',
                          'password', 'parallel', 'no-pty', 'reject-unknown-hosts', 'skip-bad-hosts', 'timeout',
                          'command-timeout', 'user', 'warn-only', 'pool-size']


def check_output(command, shell=False):
    return subprocess.check_output(command, shell=shell, executable=getattr(settings, 'SHELL', '/bin/sh'))


def check_output_with_ssh_key(command):
    if getattr(settings, 'GIT_SSH_KEY_LOCATION', None):
        return check_output('ssh-agent bash -c "ssh-add {};{}"'.format(settings.GIT_SSH_KEY_LOCATION, command),
                            shell=True)
    else:
        return check_output([command], shell=True)


def update_project_git(project, cache_dir, repo_dir):
    if not os.path.exists(repo_dir):
        if not os.path.exists(cache_dir):
            os.makedirs(cache_dir)

        check_output_with_ssh_key('git clone {} {}'.format(project.repo_url, repo_dir))
    else:
        check_output_with_ssh_key(
            'cd {0};git stash;git pull'.format(repo_dir)
        )


def setup_virtual_env_if_needed(repo_dir):
    env_dir = os.path.join(repo_dir, 'env')
    if not os.path.exists(env_dir):
        os.makedirs(env_dir)
        check_output("virtualenv {}".format(env_dir), shell=True)


def update_project_requirements(project, repo_dir, activate_loc):
    pip_installs = ' '.join(project.fabfile_requirements.splitlines())

    check_output_with_ssh_key('source {} && cd {};pip install {}'.format(activate_loc, repo_dir, pip_installs))


def get_fabfile_path(project):
    if project.use_repo_fabfile:
        cache_key = 'project_{}_fabfile_path'.format(project.pk)
        cached_result = cache.get(cache_key)

        if cached_result:
            return cached_result

        cache_dir = os.path.join(settings.PUBLIC_DIR, '.repo_caches')
        repo_dir = os.path.join(cache_dir, slugify(project.name))

        update_project_git(project, cache_dir, repo_dir)
        setup_virtual_env_if_needed(repo_dir)
        activate_loc = os.path.join(repo_dir, 'env', 'bin', 'activate')

{'fraction_non_alphanumeric': 0.1249482829954489, 'fraction_numerical': 0.0004137360364087712, 'mean_word_length': 3.5656279508970727, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2106558', 'n_tokens_mistral': 1597, 'n_tokens_neox': 1542, 'n_words': 294}