<commit_before>from fabric.api import *
from fabric.contrib.files import *
from fabric.contrib.project import rsync_project
from subprocess import check_output


env.use_ssh_config = True

env.user = 'ubuntu'

ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
HOME_DIR = '/home/ubuntu'
DEPLOY_PATH = '%s/cabot' % HOME_DIR
LOG_DIR = '/var/log/cabot/'
VENV_DIR = '%s/venv' % HOME_DIR
BACKUP_DIR = '/tmp/'

PG_DATABASE = 'index'
PG_USERNAME = 'cabot'
PG_PASSWORD = '<PASSWORD>'  # You should probably change this


def _ensure_dirs():
    dirs = [LOG_DIR]
    for d in dirs:
        sudo('mkdir -p {d}'.format(d=d))
        sudo('chmod -R 777 {d}'.format(d=d))


def _setup_venv():
    with settings(warn_only=True):
        venv_doesnt_exist = sudo('test -d %s' % VENV_DIR).failed
    if venv_doesnt_exist:
        # without --setuptools, you get `pkg_resources not found` error
        sudo('virtualenv --setuptools %s' % VENV_DIR, user='ubuntu')


def install_requirements(deploy_path=DEPLOY_PATH):
    sudo("foreman run -e conf/{env}.env {venv}/bin/pip install --editable {path} --exists-action=w".format(
        env=env.deploy_version, venv=VENV_DIR, path=deploy_path))


def run_migrations(deploy_path=DEPLOY_PATH):
    with cd(deploy_path):
        with prefix("source {venv}/bin/activate".format(venv=VENV_DIR)):
            sudo(
                "foreman run -e conf/{env}.env python manage.py syncdb --noinput".format(env=env.deploy_version))
            sudo(
                "foreman run -e conf/{env}.env python manage.py migrate cabotapp --noinput".format(env=env.deploy_version))
            sudo(
                "foreman run -e conf/{env}.env python manage.py createsuperuser".format(env=env.deploy_version))
            # Wrap in failure for legacy reasons
            # https://github.com/celery/django-celery/issues/149
            print "You can ignore an error message regarding 'relation \"celery_taskmeta\" already exists'"
            with settings(warn_only=True):
                sudo(
                    "foreman run -e conf/{env}.env python manage.py migrate djcelery --noinput".format(env=env.deploy_version))


def create_user(username, password, email):
    """ creates a django user on the cabot server.  existing users
        are clobbered so this also functions as a crude password reset.
    """
    code = (
        """username='{username}';"""
        """password='{password}';"""
        """email='{email}';"""
        """from django.contrib.auth.models import User;"""
        """User.objects.filter(username=username).delete();"""
        """User.objects.create_superuser(username, email, password);""")
    code = code.format(username=username, password=password, email=email)
    shell_cmd = "foreman run -e conf/production.env python manage.py shell"
    with prefix("source ~ubuntu/venv/bin/activate"):
        with cd("~ubuntu/cabot"):
            sudo('printf "{0}"|{1}'.format(code, shell_cmd))



def collect_static(deploy_path=DEPLOY_PATH):
    with cd(deploy_path):
        with prefix("source {venv}/bin/activate".format(venv=VENV_DIR)):
            sudo(
                "foreman run -e conf/{env}.env python manage.py collectstatic --noinput".format(env=env.deploy_version))
            sudo(
                "foreman run -e conf/{env}.env python manage.py compress".format(env=env.deploy_version))


def setup_upstart(deploy_path=DEPLOY_PATH):
    with cd(deploy_path):
        # Point at master (i.e. symlinked) path
        procfile = os.path.join(DEPLOY_PATH, 'Procfile')
        env_file = os.path.join(DEPLOY_PATH, 'conf', '%s.env' %
                                env.deploy_version)
        template_file = os.path.join(DEPLOY_PATH, 'upstart')
        sudo('foreman export upstart /etc/init -f {conf} -e {env} -u ubuntu -a cabot -t {tmplt}'.format(
            conf=procfile, env=env_file, tmplt=template_file))


def production():
    """
    Select production instance(s)
    """
    env.hosts = ['cabot.arachnys.com']


def restart():
    with settings(warn_only=True):
        restart_failed = sudo('restart cabot').failed
    if restart_failed:
        sudo('start cabot')


def stop():
    with settings(warn_only=True):
        sudo('stop cabot')


def provision():
    """
    Provision a clean Ubuntu 12.04 instance with dependencies
    """
    with open(os.path.expanduser('~/.ssh/id_rsa.pub')) as f:
        local_ssh_key = f.read().strip('\n')
    put('bin/setup_dependencies.sh', '/tmp/setup_dependencies.sh')
    sudo('LOCAL_SSH_KEY="%s" bash /tmp/setup_dependencies.sh' % local_ssh_key)
    # Clean up
    run('rm /tmp/setup_dependencies.sh')


def deploy(deploy_version=None):
    """
      Deploy a new version of code to production or test server.

      Push code to remote server, install requirements, apply migrations,
      collect and compress static assets, export foreman to upstart,
      restart service
    """
    # TODO: replace this with
    # - zip up working directory
    # - upload and unzip into DEPLOY_PATH
    env.deploy_version = deploy_version or 'production'
    dirname = check_output(
        ["echo \"$(date +'%Y-%m-%d')-$(git log --pretty=format:'%h' -n 1)\""], shell=True).strip('\n ')
    deploy_path = os.path.join(HOME_DIR, dirname)
    run('mkdir -p {}'.format(deploy_path))
    print 'Uploading project to %s' % deploy_path
    rsync_project(
        remote_dir=deploy_path,
        local_dir='./',
        ssh_opts='-o StrictHostKeyChecking=no',
        exclude=['.git', 'backups', 'venv',
                 'static/CACHE', '.vagrant', '*.pyc', 'dev.db'],
    )
    with cd(deploy_path):
        _ensure_dirs()<commit_msg>do not exclude git because in setup.py requires it<commit_after>from fabric.api import *
from fabric.contrib.files import *
from fabric.contrib.project import rsync_project
from subprocess import check_output


env.use_ssh_config = True

env.user = 'ubuntu'

ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
HOME_DIR = '/home/ubuntu'
DEPLOY_PATH = '%s/cabot' % HOME_DIR
LOG_DIR = '/var/log/cabot/'
VENV_DIR = '%s/venv' % HOME_DIR
BACKUP_DIR = '/tmp/'

PG_DATABASE = 'index'
PG_USERNAME = 'cabot'
PG_PASSWORD = '<PASSWORD>'  # You should probably change this


def _ensure_dirs():
    dirs = [LOG_DIR]
    for d in dirs:
        sudo('mkdir -p {d}'.format(d=d))
        sudo('chmod -R 777 {d}'.format(d=d))


def _setup_venv():
    with settings(warn_only=True):
        venv_doesnt_exist = sudo('test -d %s' % VENV_DIR).failed
    if venv_doesnt_exist:
        # without --setuptools, you get `pkg_resources not found` error
        sudo('virtualenv --setuptools %s' % VENV_DIR, user='ubuntu')


def install_requirements(deploy_path=DEPLOY_PATH):
    sudo("foreman run -e conf/{env}.env {venv}/bin/pip install --editable {path} --exists-action=w".format(
        env=env.deploy_version, venv=VENV_DIR, path=deploy_path))


def run_migrations(deploy_path=DEPLOY_PATH):
    with cd(deploy_path):
        with prefix("source {venv}/bin/activate".format(venv=VENV_DIR)):
            sudo(
                "foreman run -e conf/{env}.env python manage.py syncdb --noinput".format(env=env.deploy_version))
            sudo(
                "foreman run -e conf/{env}.env python manage.py migrate cabotapp --noinput".format(env=env.deploy_version))
            sudo(
                "foreman run -e conf/{env}.env python manage.py createsuperuser".format(env=env.deploy_version))
            # Wrap in failure for legacy reasons
            # https://github.com/celery/django-celery/issues/149
            print "You can ignore an error message regarding 'relation \"celery_taskmeta\" already exists'"
            with settings(warn_only=True):
                sudo(
                    "foreman run -e conf/{env}.env python manage.py migrate djcelery --noinput".format(env=env.deploy_version))


def create_user(username, password, email):
    """ creates a django user on the cabot server.  existing users
        are clobbered so this also functions as a crude password reset.
    """
    code = (
        """username='{username}';"""
        """password='{password}';"""
        """email='{email}';"""
        """from django.contrib.auth.models import User;"""
        """User.objects.filter(username=username).delete();"""
        """User.objects.create_superuser(username, email, password);""")
    code = code.format(username=username, password=password, email=email)
    shell_cmd = "foreman run -e conf/production.env python manage.py shell"
    with prefix("source ~ubuntu/venv/bin/activate"):
        with cd("~ubuntu/cabot"):
            sudo('printf "{0}"|{1}'.format(code, shell_cmd))



def collect_static(deploy_path=DEPLOY_PATH):
    with cd(deploy_path):
        with prefix("source {venv}/bin/activate".format(venv=VENV_DIR)):
            sudo(
                "foreman run -e conf/{env}.env python manage.py collectstatic --noinput".format(env=env.deploy_version))
            sudo(
                "foreman run -e conf/{env}.env python manage.py compress".format(env=env.deploy_version))


def setup_upstart(deploy_path=DEPLOY_PATH):
    with cd(deploy_path):
        # Point at master (i.e. symlinked) path
        procfile = os.path.join(DEPLOY_PATH, 'Procfile')
        env_file = os.path.join(DEPLOY_PATH, 'conf', '%s.env' %
                                env.deploy_version)
        template_file = os.path.join(DEPLOY_PATH, 'upstart')
        sudo('foreman export upstart /etc/init -f {conf} -e {env} -u ubuntu -a cabot -t {tmplt}'.format(
            conf=procfile, env=env_file, tmplt=template_file))


def production():
    """
    Select production instance(s)
    """
    env.hosts = ['cabot.arachnys.com']


def restart():
    with settings(warn_only=True):
        restart_failed = sudo('restart cabot').failed
    if restart_failed:
        sudo('start cabot')


def stop():
    with settings(warn_only=True):
        sudo('stop cabot')


def provision():
    """
    Provision a clean Ubuntu 12.04 instance with dependencies
    """
    with open(os.path.expanduser('~/.ssh/id_rsa.pub')) as f:
        local_ssh_key = f.read().strip('\n')
    put('bin/setup_dependencies.sh', '/tmp/setup_dependencies.sh')
    sudo('LOCAL_SSH_KEY="%s" bash /tmp/setup_dependencies.sh' % local_ssh_key)
    # Clean up
    run('rm /tmp/setup_dependencies.sh')


def deploy(deploy_version=None):
    """
      Deploy a new version of code to production or test server.

      Push code to remote server, install requirements, apply migrations,
      collect and compress static assets, export foreman to upstart,
      restart service
    """
    # TODO: replace this with
    # - zip up working directory
    # - upload and unzip into DEPLOY_PATH
    env.deploy_version = deploy_version or 'production'
    dirname = check_output(
        ["echo \"$(date +'%Y-%m-%d')-$(git log --pretty=format:'%h' -n 1)\""], shell=True).strip('\n ')
    deploy_path = os.path.join(HOME_DIR, dirname)
    run('mkdir -p {}'.format(deploy_path))
    print 'Uploading project to %s' % deploy_path
    rsync_project(
        remote_dir=deploy_path,
        local_dir='./',
        ssh_opts='-o StrictHostKeyChecking=no',
        exclude=['backups', 'venv',
                 'static/CACHE', '.vagrant', '*.pyc', 'dev.db'],
    )
    with cd(deploy_path):
        _ensure_dirs()
{'fraction_non_alphanumeric': 0.13815963935295678, 'fraction_numerical': 0.0022982409617254484, 'mean_word_length': 2.933936022253129, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5763843', 'n_tokens_mistral': 3797, 'n_tokens_neox': 3584, 'n_words': 840}