<commit_before>import os
import pwd
from fabric.api import cd, env, task, require, local, lcd, sudo, prefix, shell_env
from fabric.contrib.files import exists, upload_template


VIRTUALENV_DIR = 'connectedafrica_ve'
CODE_DIR = 'connectedafrica'
PROD_HOSTS = ['ec2-54-76-245-198.eu-west-1.compute.amazonaws.com']
PACKAGES = (
    'git-core',
    'memcached',
    'nginx',
    'npm',
    'python-dev',
    'python-virtualenv',
    'supervisor',
)
# Nginx & Supervisor constants
SERVER_NAMES = 'sa.connectedafrica.org ec2-54-76-245-198.eu-west-1.compute.amazonaws.com'
PROXY_PORT = 5001
PROXY_HOST = '127.0.0.1'
LOG_DIR = 'connectedafrica_logs'


@task
def prod():
    env.deploy_user = 'connectedafrica'
    env.deploy_dir = '/home/connectedafrica/'
    env.branch = 'master'
    env.hosts = PROD_HOSTS


@task
def provision():
    require('deploy_user', 'deploy_dir', provided_by=[prod])

    commands = (
        'apt-get install %s --no-upgrade' % ' '.join(PACKAGES),
        'apt-get build-dep lxml --no-upgrade',
        'mkdir -p %s' % env.deploy_dir,
        'chown -R %s:%s %s' % (env.deploy_user, env.deploy_user, env.deploy_dir),
    )
    sudo('; '.join(commands))


@task
def deploy():
    require('deploy_user', 'deploy_dir', 'branch',
            provided_by=[prod])

    repo_dir = os.path.join(env.deploy_dir, CODE_DIR)
    ve_dir = os.path.join(env.deploy_dir, VIRTUALENV_DIR)

    if not exists(ve_dir):
        sudo('virtualenv -p python2.7 %s' % ve_dir, user=env.deploy_user)

    if not exists(repo_dir):
        with cd(env.deploy_dir):
            sudo('git clone -b %s https://github.com/codeforafrica/connectedAFRICA.git %s'
                 % (env.branch, repo_dir), user=env.deploy_user)
    else:
        with cd(repo_dir):
            sudo('git checkout -B %s' % env.branch, user=env.deploy_user)
            sudo('git pull origin %s' % env.branch, user=env.deploy_user)

    with cd(repo_dir), prefix('. ../%s/bin/activate' % VIRTUALENV_DIR):
        sudo('pip install -e ./', user=env.deploy_user)
        sudo('make install', user=env.deploy_user)

    # render and upload templates
    upload_template(os.path.join(os.path.dirname(__file__), 'conf/nginx.template'),
                    '/etc/nginx/sites-enabled/connectedafrica.conf',
                    get_nginx_template_context(), use_sudo=True, backup=False)
    upload_template(os.path.join(os.path.dirname(__file__), 'conf/supervisor.template'),
                    '/etc/supervisor/conf.d/connectedafrica.conf',
                    get_supervisor_template_context(), use_sudo=True, backup=False)
    # make sure logging dir exists and update processes
    log_dir = os.path.join(env.deploy_dir, LOG_DIR)
    sudo('mkdir -p %s' % log_dir, user=env.deploy_user)
    sudo('supervisorctl restart connectedafrica:*')
    sudo('/etc/init.d/nginx reload')


def get_nginx_template_context():
    return {
        'server-name': SERVER_NAMES,
        'server-port': 80,
        'static-path': os.path.join(env.deploy_dir, 'connectedafrica/connectedafrica/static/'),
        'log': os.path.join(env.deploy_dir, LOG_DIR, 'nginx.log'),
        'err-log': os.path.join(env.deploy_dir, LOG_DIR, 'nginx.err'),<commit_msg>fabric make task for running arbitrary make commands<commit_after>import os
import pwd
from fabric.api import cd, env, task, require, sudo, prefix, shell_env, run
from fabric.contrib.files import exists, upload_template
from fabric.context_managers import shell_env


VIRTUALENV_DIR = 'connectedafrica_ve'
CODE_DIR = 'connectedafrica'
PROD_HOSTS = ['ec2-54-76-245-198.eu-west-1.compute.amazonaws.com']
PACKAGES = (
    'git-core',
    'memcached',
    'nginx',
    'npm',
    'python-dev',
    'python-virtualenv',
    'supervisor',
)
# Nginx & Supervisor constants
SERVER_NAMES = 'sa.connectedafrica.org ec2-54-76-245-198.eu-west-1.compute.amazonaws.com'
PROXY_PORT = 5001
PROXY_HOST = '127.0.0.1'
LOG_DIR = 'connectedafrica_logs'


@task
def prod():
    env.deploy_user = 'connectedafrica'
    env.deploy_dir = '/home/connectedafrica/'
    env.branch = 'master'
    env.hosts = PROD_HOSTS


@task
def provision():
    require('deploy_user', 'deploy_dir', provided_by=[prod])

    commands = (
        'apt-get install %s --no-upgrade' % ' '.join(PACKAGES),
        'apt-get build-dep lxml --no-upgrade',
        'mkdir -p %s' % env.deploy_dir,
        'chown -R %s:%s %s' % (env.deploy_user, env.deploy_user, env.deploy_dir),
    )
    sudo('; '.join(commands))


@task
def deploy():
    require('deploy_user', 'deploy_dir', 'branch',
            provided_by=[prod])

    repo_dir = os.path.join(env.deploy_dir, CODE_DIR)
    ve_dir = os.path.join(env.deploy_dir, VIRTUALENV_DIR)

    if not exists(ve_dir):
        sudo('virtualenv -p python2.7 %s' % ve_dir, user=env.deploy_user)

    if not exists(repo_dir):
        with cd(env.deploy_dir):
            sudo('git clone -b %s https://github.com/codeforafrica/connectedAFRICA.git %s'
                 % (env.branch, repo_dir), user=env.deploy_user)
    else:
        with cd(repo_dir):
            sudo('git checkout -B %s' % env.branch, user=env.deploy_user)
            sudo('git pull origin %s' % env.branch, user=env.deploy_user)

    with cd(repo_dir), prefix('. ../%s/bin/activate' % VIRTUALENV_DIR):
        sudo('pip install -e ./', user=env.deploy_user)
        sudo('make install', user=env.deploy_user)

    # render and upload templates
    upload_template(os.path.join(os.path.dirname(__file__), 'conf/nginx.template'),
                    '/etc/nginx/sites-enabled/connectedafrica.conf',
                    get_nginx_template_context(), use_sudo=True, backup=False)
    upload_template(os.path.join(os.path.dirname(__file__), 'conf/supervisor.template'),
                    '/etc/supervisor/conf.d/connectedafrica.conf',
                    get_supervisor_template_context(), use_sudo=True, backup=False)
    # make sure logging dir exists and update processes
    log_dir = os.path.join(env.deploy_dir, LOG_DIR)
    sudo('mkdir -p %s' % log_dir, user=env.deploy_user)
    sudo('supervisorctl update')
    sudo('/etc/init.d/nginx reload')


@task
def make(rule):
    require('deploy_dir', provided_by=[prod])

    repo_dir = os.path.join(env.deploy_dir, CODE_DIR)

    with cd(repo_dir), prefix('. ../%s/bin/activate' % VIRTUALENV_DIR):
        with shell_env(GRANO_HOST='http://localhost:9000',
                       GRANO_PROJECT='connectedafrica-sa',
                       COAF_SETTINGS='settings'):
            run('make %s' % rule)


def get_nginx_template_context():
    return {
        'server-name': SERVER_NAMES,
        'server-port': 80,
        'static-path': os.path.join(env.deploy_dir, 'connectedafrica/connectedafrica/static/'),
        'log': os.path.join(env.deploy_dir, LOG_DIR, 'nginx.log'),
        'err-log': os.path.join(env.deploy_dir, LOG_DIR, 'nginx.err'),
{'fraction_non_alphanumeric': 0.1453802364618304, 'fraction_numerical': 0.011677127426653043, 'mean_word_length': 3.0688836104513064, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '440852', 'n_tokens_mistral': 2515, 'n_tokens_neox': 2336, 'n_words': 438}