<commit_before>import os
import time
from fabric.api import env, execute, lcd, local, put, run, roles, task

import commander.hosts
import commander_settings as settings


env.key_filename = settings.SSH_KEY
env.roledefs.update(commander.hosts.hostgroups)

_src_dir = lambda *p: os.path.join(settings.SRC_DIR, *p)
ROOT_DIR = os.path.dirname(settings.SRC_DIR)
VIRTUALENV = os.path.join(ROOT_DIR, 'venv')

BUILD_ID = str(int(time.time()))
INSTALL_TO = os.path.join(os.path.dirname(settings.WWW_DIR), BUILD_ID)
PACKAGE_DIR = '/tmp'


def setup_notifier():
    notifier_endpoint = getattr(settings, 'NOTIFIER_ENDPOINT', None)
    notifier_key = getattr(settings, 'NOTIFIER_KEY', None)
    null_notify = lambda a: None
    if not notifier_endpoint:
        return null_notify

    try:
        import pushbotnotify
    except ImportError:
        return null_notify

    notifier = pushbotnotify.Notifier(endpoint=notifier_endpoint,
                                      api_key=notifier_key)

    return notifier.notify


notify = setup_notifier()


@task
def create_virtualenv():
    with lcd(settings.SRC_DIR):
        status = local('git diff HEAD@{1} HEAD --name-only')

    if 'requirements/' in status:
        venv = VIRTUALENV
        if not venv.startswith('/data'):
            raise Exception('venv must start with /data')

        local('rm -rf %s' % venv)
        local('virtualenv --distribute --never-download %s' % venv)

        local('%s/bin/pip install --exists-action=w --no-deps --no-index '
              '--download-cache=/tmp/pip-cache -f %s '
              '-r %s/requirements/prod.txt' %
              (venv, settings.PYREPO, settings.SRC_DIR))

        if getattr(settings, 'LOAD_TESTING', False):
            local('%s/bin/pip install --exists-action=w --no-deps '
                  '--no-index --download-cache=/tmp/pip-cache -f %s '
                  '-r %s/requirements/load.txt' %
                  (venv, settings.PYREPO, settings.SRC_DIR))

        # make sure this always runs
        local("rm -f %s/lib/python2.6/no-global-site-packages.txt" % venv)
        local("%s/bin/python /usr/bin/virtualenv --relocatable %s" %
              (venv, venv))


@task
def update_locales():
    with lcd(_src_dir("locale")):
        local("svn revert -R .")
        local("svn up")
        local("./compile-mo.sh .")


@task
def loadtest(repo=''):
    if hasattr(settings, 'MARTEAU'):
        os.environ['MACAUTH_USER'] = settings.MARTEAU_USER
        os.environ['MACAUTH_SECRET'] = settings.MARTEAU_SECRET
        local('%s %s --server %s' % (settings.MARTEAU, repo,
                                     settings.MARTEAU_SERVER))


@task
def update_products():
    with lcd(settings.SRC_DIR):
        local('%s manage.py update_product_details' % settings.PYTHON)


@task
def compress_assets(arg=''):
    with lcd(settings.SRC_DIR):
        local("%s manage.py compress_assets -t %s" % (settings.PYTHON,
                                                      arg))


@task
def schematic():
    with lcd(settings.SRC_DIR):
        local("%s %s/bin/schematic migrations" %
              (settings.PYTHON, VIRTUALENV))


@task
def update_code(ref='origin/master'):
    with lcd(settings.SRC_DIR):
        local("git fetch && git fetch -t")
        local("git reset --hard %s" % ref)
        local("git submodule sync")
        local("git submodule update --init --recursive")
        # Recursively run submodule sync/update to get all the right repo URLs.
        local("git submodule foreach 'git submodule sync --quiet'")
        local("git submodule foreach "
              "'git submodule update --init --recursive'")


@task
def update_info(ref='origin/master'):
    with lcd(settings.SRC_DIR):
        local("git status")
        local("git log -1")
        local("/bin/bash -c "
              "'source /etc/bash_completion.d/git && __git_ps1'")
        local('git show -s {0} --pretty="format:%h" '
              '> media/git-rev.txt'.format(ref))


@task
def checkin_changes():
    local(settings.DEPLOY_SCRIPT)


@task
def build_package():
    with lcd(settings.SRC_DIR):
        ref = local('git rev-parse HEAD', capture=True)
    ref = ref[:6]

    PACKAGE_NAME = 'zamboni-%s-%s-%s' % (getattr(settings, 'ENV', 'dev'),
                                         ref, BUILD_ID)
    PACKAGE_FILE = os.path.join(PACKAGE_DIR, '%s.rpm' % PACKAGE_NAME)
    local('fpm -s dir -t rpm -n "%s" '
          '-p "%s" '
          '-x "*.git" -x "*.svn" -x "*.pyc" '
          '-C %s --prefix "%s" zamboni venv' % (PACKAGE_NAME,
                                                PACKAGE_FILE,
                                                ROOT_DIR,
                                                INSTALL_TO))

    return PACKAGE_FILE


@roles(settings.WEB_HOSTGROUP, settings.CELERY_HOSTGROUP)
@task
def install_package(package_file):
    put(package_file, package_file)
    run('rpm -i %s' % package_file)


@task
def disable_cron():
    local("rm -f /etc/cron.d/%s" % settings.CRON_NAME)


@task
def install_cron():
    with lcd(settings.SRC_DIR):
        local('%s ./scripts/crontab/gen-cron.py '
              '-z %s -r %s/bin -u apache -p %s > /etc/cron.d/.%s' %
              (settings.PYTHON, settings.SRC_DIR, settings.REMORA_DIR,
               settings.PYTHON, settings.CRON_NAME))<commit_msg>Flip symlink on package install.<commit_after>import os
import time
from fabric.api import env, execute, lcd, local, put, run, roles, task

import commander.hosts
import commander_settings as settings


env.key_filename = settings.SSH_KEY
env.roledefs.update(commander.hosts.hostgroups)

_src_dir = lambda *p: os.path.join(settings.SRC_DIR, *p)
ROOT_DIR = os.path.dirname(settings.SRC_DIR)
VIRTUALENV = os.path.join(ROOT_DIR, 'venv')

BUILD_ID = str(int(time.time()))
INSTALL_TO = os.path.join(os.path.dirname(settings.WWW_DIR), BUILD_ID)
PACKAGE_DIR = '/tmp'


def setup_notifier():
    notifier_endpoint = getattr(settings, 'NOTIFIER_ENDPOINT', None)
    notifier_key = getattr(settings, 'NOTIFIER_KEY', None)
    null_notify = lambda a: None
    if not notifier_endpoint:
        return null_notify

    try:
        import pushbotnotify
    except ImportError:
        return null_notify

    notifier = pushbotnotify.Notifier(endpoint=notifier_endpoint,
                                      api_key=notifier_key)

    return notifier.notify


notify = setup_notifier()


@task
def create_virtualenv():
    with lcd(settings.SRC_DIR):
        status = local('git diff HEAD@{1} HEAD --name-only')

    if 'requirements/' in status:
        venv = VIRTUALENV
        if not venv.startswith('/data'):
            raise Exception('venv must start with /data')

        local('rm -rf %s' % venv)
        local('virtualenv --distribute --never-download %s' % venv)

        local('%s/bin/pip install --exists-action=w --no-deps --no-index '
              '--download-cache=/tmp/pip-cache -f %s '
              '-r %s/requirements/prod.txt' %
              (venv, settings.PYREPO, settings.SRC_DIR))

        if getattr(settings, 'LOAD_TESTING', False):
            local('%s/bin/pip install --exists-action=w --no-deps '
                  '--no-index --download-cache=/tmp/pip-cache -f %s '
                  '-r %s/requirements/load.txt' %
                  (venv, settings.PYREPO, settings.SRC_DIR))

        # make sure this always runs
        local("rm -f %s/lib/python2.6/no-global-site-packages.txt" % venv)
        local("%s/bin/python /usr/bin/virtualenv --relocatable %s" %
              (venv, venv))


@task
def update_locales():
    with lcd(_src_dir("locale")):
        local("svn revert -R .")
        local("svn up")
        local("./compile-mo.sh .")


@task
def loadtest(repo=''):
    if hasattr(settings, 'MARTEAU'):
        os.environ['MACAUTH_USER'] = settings.MARTEAU_USER
        os.environ['MACAUTH_SECRET'] = settings.MARTEAU_SECRET
        local('%s %s --server %s' % (settings.MARTEAU, repo,
                                     settings.MARTEAU_SERVER))


@task
def update_products():
    with lcd(settings.SRC_DIR):
        local('%s manage.py update_product_details' % settings.PYTHON)


@task
def compress_assets(arg=''):
    with lcd(settings.SRC_DIR):
        local("%s manage.py compress_assets -t %s" % (settings.PYTHON,
                                                      arg))


@task
def schematic():
    with lcd(settings.SRC_DIR):
        local("%s %s/bin/schematic migrations" %
              (settings.PYTHON, VIRTUALENV))


@task
def update_code(ref='origin/master'):
    with lcd(settings.SRC_DIR):
        local("git fetch && git fetch -t")
        local("git reset --hard %s" % ref)
        local("git submodule sync")
        local("git submodule update --init --recursive")
        # Recursively run submodule sync/update to get all the right repo URLs.
        local("git submodule foreach 'git submodule sync --quiet'")
        local("git submodule foreach "
              "'git submodule update --init --recursive'")


@task
def update_info(ref='origin/master'):
    with lcd(settings.SRC_DIR):
        local("git status")
        local("git log -1")
        local("/bin/bash -c "
              "'source /etc/bash_completion.d/git && __git_ps1'")
        local('git show -s {0} --pretty="format:%h" '
              '> media/git-rev.txt'.format(ref))


@task
def checkin_changes():
    local(settings.DEPLOY_SCRIPT)


@task
def build_package():
    with lcd(settings.SRC_DIR):
        ref = local('git rev-parse HEAD', capture=True)
    ref = ref[:6]

    PACKAGE_NAME = 'zamboni-%s-%s-%s' % (getattr(settings, 'ENV', 'dev'),
                                         ref, BUILD_ID)
    PACKAGE_FILE = os.path.join(PACKAGE_DIR, '%s.rpm' % PACKAGE_NAME)
    local('fpm -s dir -t rpm -n "%s" '
          '-p "%s" '
          '-x "*.git" -x "*.svn" -x "*.pyc" '
          '-C %s --prefix "%s" zamboni venv' % (PACKAGE_NAME,
                                                PACKAGE_FILE,
                                                ROOT_DIR,
                                                INSTALL_TO))

    return PACKAGE_FILE


@roles(settings.WEB_HOSTGROUP, settings.CELERY_HOSTGROUP)
@task
def install_package(package_file):
    cur_sym = os.path.join(os.path.dirname(INSTALL_TO), 'current')

    put(package_file, package_file)
    run('rpm -i %s' % package_file)
    run('[[ -d {0} ]] && ln -s {0} {1}'.format(INSTALL_TO, cur_sym))


@task
def disable_cron():
    local("rm -f /etc/cron.d/%s" % settings.CRON_NAME)


@task
def install_cron():
    with lcd(settings.SRC_DIR):
        local('%s ./scripts/crontab/gen-cron.py '
              '-z %s -r %s/bin -u apache -p %s > /etc/cron.d/.%s' %
              (settings.PYTHON, settings.SRC_DIR, settings.REMORA_DIR,
               settings.PYTHON, settings.CRON_NAME))
{'fraction_non_alphanumeric': 0.13823474874837752, 'fraction_numerical': 0.001576117188948637, 'mean_word_length': 2.212328767123288, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3679620', 'n_tokens_mistral': 3619, 'n_tokens_neox': 3477, 'n_words': 766}