<commit_before>import os
import pwd
from fabric.api import cd, env, task, require, local, lcd, sudo, prefix, shell_env
from fabric.contrib.files import exists, upload_template

from api.fabfile import (provision_api, create_api_database, drop_api_database,
    load_api_data)


VIRTUALENV_DIR = 'censusreporter_ve'
CODE_DIR = 'censusreporter'
PROD_HOSTS = ['wazimap.co.za']
PACKAGES = (
    'git-core',
    'python-virtualenv',
    'sqlite3',
    'memcached',
    'upstart',
    'nginx',
    'npm'  # Node is for yUglify
)
# Nginx & Upstart constants
SERVER_NAMES = 'wazimap.co.za'
EMBED_URL = 'embed.wazimap.co.za'
SERVER_ALIASES = 'wazimap.com wazimap.org wazimap.net wazimap.info mma-dashboard.code4sa.org'
PROXY_PORT = 5001
PROXY_HOST = '127.0.0.1'
LOG_DIR = 'censusreporter_logs'


@task
def dev():
    env.deploy_type = 'dev'
    env.deploy_user = pwd.getpwuid(os.getuid())[0]
    env.deploy_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), '../')
    env.branch = None  # deploy whichever branch we are currently on


@task
def prod():
    env.deploy_type = 'prod'
    env.deploy_user = 'www-data'
    env.deploy_dir = '/var/www-data/'
    env.branch = 'master'
    env.hosts = PROD_HOSTS
    env.user = 'mma'


@task
def provision():
    require('deploy_type', provided_by=[dev, prod])

    commands = (
        'apt-get install %s --no-upgrade' % ' '.join(PACKAGES),
    )
    if env.deploy_type == 'prod':
        commands += (
            'mkdir -p %s' % env.deploy_dir,
            'chown -R %s:%s %s' % (env.deploy_user, env.deploy_user, env.deploy_dir),
        )

    if env.deploy_type == 'dev':
        local('; '.join('sudo %s' % cmd for cmd in commands))
        local('sudo apt-get build-dep python-numpy python-psycopg2 --no-upgrade')
        local('sudo npm config set registry http://registry.npmjs.org/')
        local('sudo npm -g install yuglify')
        return
    sudo('; '.join(commands))
    sudo('apt-get build-dep python-numpy python-psycopg2 --no-upgrade')
    sudo('npm config set registry http://registry.npmjs.org/')
    sudo('npm -g install yuglify')


@task
def deploy():
    require('deploy_type', 'deploy_user', 'deploy_dir', 'branch',
            provided_by=[dev, prod])

    repo_dir = os.path.join(env.deploy_dir, CODE_DIR)
    ve_dir = os.path.join(env.deploy_dir, VIRTUALENV_DIR)

    if env.deploy_type == 'dev':
        with lcd(repo_dir):
            local('git pull')
            local('. ~/.virtualenvs/%s/bin/activate && pip install -r requirements.txt'
                  % CODE_DIR)
        return

    if not exists(ve_dir):
        sudo('virtualenv -p python2.7 %s' % ve_dir, user=env.deploy_user)

    if not exists(repo_dir):
        with cd(env.deploy_dir):
            sudo('git clone -b %s https://github.com/Code4SA/censusreporter.git'
                 % env.branch, user=env.deploy_user)
    else:
        with cd(repo_dir):
            sudo('git checkout -B %s' % env.branch, user=env.deploy_user)
            sudo('git pull origin %s' % env.branch, user=env.deploy_user)

    # upload pth file to manage python path
    upload_template(os.path.join(os.path.dirname(__file__), 'conf/pythonpath.template'),
                    '%s/lib/python2.7/site-packages/censusreporter.pth' % ve_dir,
                    {'project-dir': repo_dir}, use_sudo=True, backup=False)

    with cd(repo_dir), prefix('. ../%s/bin/activate' % VIRTUALENV_DIR):
        sudo('pip install -r requirements.txt', user=env.deploy_user)
        sudo('python manage.py collectstatic --settings=config.prod.settings '
             '--noinput', user=env.deploy_user)

    # render and upload templates
    upload_template(os.path.join(os.path.dirname(__file__), 'conf/nginx.template'),
                    '/etc/nginx/sites-enabled/censusreporter.conf',
                    get_nginx_template_context(), use_sudo=True, backup=False)
    upload_template(os.path.join(os.path.dirname(__file__), 'conf/upstart.template'),
                    '/etc/init/censusreporter.conf',
                    get_upstart_template_context(), use_sudo=True, backup=False)
    # make sure logging dir exists and update processes
    log_dir = os.path.join(env.deploy_dir, LOG_DIR)
    sudo('mkdir -p %s' % log_dir, user=env.deploy_user)

    cache_dir = '/var/tmp/censusreporter_cache/'
    sudo('mkdir -p %s' % cache_dir, user=env.deploy_user)

    embed_dir = os.path.join(env.deploy_dir, 'censusreporter/embed_data/profiles/')
    sudo('mkdir -p %s' % embed_dir, user=env.deploy_user)

    sudo('initctl restart censusreporter')
    sudo('/etc/init.d/nginx reload')


def get_nginx_template_context():
    return {
        'server-name': SERVER_NAMES,
        'embed-server-name': EMBED_URL,
        'server-aliases': SERVER_ALIASES,
        'server-port': 80,
        'static-path': os.path.join(env.deploy_dir, 'censusreporter/censusreporter/static/'),
        'embed-data-path': os.path.join(env.deploy_dir, 'censusreporter/censusreporter/embed_data/'),
        'log': os.path.join(env.deploy_dir, LOG_DIR, 'nginx.log'),
        'err-log': os.path.join(env.deploy_dir, LOG_DIR, 'nginx.err'),
        'proxy-host': PROXY_HOST,
        'proxy-port': PROXY_PORT,
    }

<commit_msg>Hotfix: configure Nginx to serve /embed_data/ from /var/www-data/censusreporter/embed_data/.<commit_after>import os
import pwd
from fabric.api import cd, env, task, require, local, lcd, sudo, prefix, shell_env
from fabric.contrib.files import exists, upload_template

from api.fabfile import (provision_api, create_api_database, drop_api_database,
    load_api_data)


VIRTUALENV_DIR = 'censusreporter_ve'
CODE_DIR = 'censusreporter'
PROD_HOSTS = ['wazimap.co.za']
PACKAGES = (
    'git-core',
    'python-virtualenv',
    'sqlite3',
    'memcached',
    'upstart',
    'nginx',
    'npm'  # Node is for yUglify
)
# Nginx & Upstart constants
SERVER_NAMES = 'wazimap.co.za'
EMBED_URL = 'embed.wazimap.co.za'
SERVER_ALIASES = 'wazimap.com wazimap.org wazimap.net wazimap.info mma-dashboard.code4sa.org'
PROXY_PORT = 5001
PROXY_HOST = '127.0.0.1'
LOG_DIR = 'censusreporter_logs'


@task
def dev():
    env.deploy_type = 'dev'
    env.deploy_user = pwd.getpwuid(os.getuid())[0]
    env.deploy_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), '../')
    env.branch = None  # deploy whichever branch we are currently on


@task
def prod():
    env.deploy_type = 'prod'
    env.deploy_user = 'www-data'
    env.deploy_dir = '/var/www-data/'
    env.branch = 'master'
    env.hosts = PROD_HOSTS
    env.user = 'mma'


@task
def provision():
    require('deploy_type', provided_by=[dev, prod])

    commands = (
        'apt-get install %s --no-upgrade' % ' '.join(PACKAGES),
    )
    if env.deploy_type == 'prod':
        commands += (
            'mkdir -p %s' % env.deploy_dir,
            'chown -R %s:%s %s' % (env.deploy_user, env.deploy_user, env.deploy_dir),
        )

    if env.deploy_type == 'dev':
        local('; '.join('sudo %s' % cmd for cmd in commands))
        local('sudo apt-get build-dep python-numpy python-psycopg2 --no-upgrade')
        local('sudo npm config set registry http://registry.npmjs.org/')
        local('sudo npm -g install yuglify')
        return
    sudo('; '.join(commands))
    sudo('apt-get build-dep python-numpy python-psycopg2 --no-upgrade')
    sudo('npm config set registry http://registry.npmjs.org/')
    sudo('npm -g install yuglify')


@task
def deploy():
    require('deploy_type', 'deploy_user', 'deploy_dir', 'branch',
            provided_by=[dev, prod])

    repo_dir = os.path.join(env.deploy_dir, CODE_DIR)
    ve_dir = os.path.join(env.deploy_dir, VIRTUALENV_DIR)

    if env.deploy_type == 'dev':
        with lcd(repo_dir):
            local('git pull')
            local('. ~/.virtualenvs/%s/bin/activate && pip install -r requirements.txt'
                  % CODE_DIR)
        return

    if not exists(ve_dir):
        sudo('virtualenv -p python2.7 %s' % ve_dir, user=env.deploy_user)

    if not exists(repo_dir):
        with cd(env.deploy_dir):
            sudo('git clone -b %s https://github.com/Code4SA/censusreporter.git'
                 % env.branch, user=env.deploy_user)
    else:
        with cd(repo_dir):
            sudo('git checkout -B %s' % env.branch, user=env.deploy_user)
            sudo('git pull origin %s' % env.branch, user=env.deploy_user)

    # upload pth file to manage python path
    upload_template(os.path.join(os.path.dirname(__file__), 'conf/pythonpath.template'),
                    '%s/lib/python2.7/site-packages/censusreporter.pth' % ve_dir,
                    {'project-dir': repo_dir}, use_sudo=True, backup=False)

    with cd(repo_dir), prefix('. ../%s/bin/activate' % VIRTUALENV_DIR):
        sudo('pip install -r requirements.txt', user=env.deploy_user)
        sudo('python manage.py collectstatic --settings=config.prod.settings '
             '--noinput', user=env.deploy_user)

    # render and upload templates
    upload_template(os.path.join(os.path.dirname(__file__), 'conf/nginx.template'),
                    '/etc/nginx/sites-enabled/censusreporter.conf',
                    get_nginx_template_context(), use_sudo=True, backup=False)
    upload_template(os.path.join(os.path.dirname(__file__), 'conf/upstart.template'),
                    '/etc/init/censusreporter.conf',
                    get_upstart_template_context(), use_sudo=True, backup=False)
    # make sure logging dir exists and update processes
    log_dir = os.path.join(env.deploy_dir, LOG_DIR)
    sudo('mkdir -p %s' % log_dir, user=env.deploy_user)

    cache_dir = '/var/tmp/censusreporter_cache/'
    sudo('mkdir -p %s' % cache_dir, user=env.deploy_user)

    embed_dir = os.path.join(env.deploy_dir, 'censusreporter/embed_data/profiles/')
    sudo('mkdir -p %s' % embed_dir, user=env.deploy_user)

    sudo('initctl restart censusreporter')
    sudo('/etc/init.d/nginx reload')


def get_nginx_template_context():
    return {
        'server-name': SERVER_NAMES,
        'embed-server-name': EMBED_URL,
        'server-aliases': SERVER_ALIASES,
        'server-port': 80,
        'static-path': os.path.join(env.deploy_dir, 'censusreporter/censusreporter/static/'),
        'embed-data-path': os.path.join(env.deploy_dir, 'censusreporter/embed_data/'),
        'log': os.path.join(env.deploy_dir, LOG_DIR, 'nginx.log'),
        'err-log': os.path.join(env.deploy_dir, LOG_DIR, 'nginx.err'),
        'proxy-host': PROXY_HOST,
        'proxy-port': PROXY_PORT,
    }


{'fraction_non_alphanumeric': 0.14297977871041587, 'fraction_numerical': 0.004196871423120946, 'mean_word_length': 2.8977695167286246, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 5, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5025500', 'n_tokens_mistral': 3842, 'n_tokens_neox': 3611, 'n_words': 703}