<commit_before>"""
Deploy this project in stage/production.

Requires commander_ which is installed on the systems that need it.

.. _commander: https://github.com/oremj/commander
"""

import os
import requests
import sys

from commander.deploy import hostgroups, task

sys.path.append(os.path.dirname(os.path.abspath(__file__)))
import commander_settings as settings  # noqa

env_file = os.path.join(settings.SRC_DIR, 'treeherder-env.sh')
th_service_src = os.path.join(settings.SRC_DIR, 'treeherder-service')

is_prod = 'treeherder.mozilla.org' in settings.SRC_DIR


def run_local_with_env(ctx, cmd):
    # For commands run from the admin node, we have to manually set the environment
    # variables, since the admin node is shared by both stage and prod.
    ctx.local("source {} && {}".format(env_file, cmd))


@task
def pre_update(ctx, ref=settings.UPDATE_REF):
    # Update the code to a specific git reference (branch/tag/sha) and write
    # info about the current repository state to a publicly visible file.
    with ctx.lcd(th_service_src):
        ctx.local('git fetch --quiet origin %s' % ref)
        ctx.local('git reset --hard FETCH_HEAD')
        # Fail early if we've forgotten to run grunt build before deploy.
        # This is easier to do (and more necessary) - now that dist/ is not
        # present on master, and only created on the stage/production branches.
        if not os.path.isfile(os.path.join(th_service_src, 'dist', 'index.html')):
            raise Exception("Grunt build hasn't been run and committed to the repo!")
        ctx.local('find . -type f -name "*.pyc" -delete')
        # Remove gzipped UI assets in case the uncompressed original no longer exists.
        ctx.local('find dist/ -type f -name "*.gz" -delete')
        ctx.local('git status -s')
        # Make the current Git revision accessible at <site-root>/revision.txt
        ctx.local('git rev-parse HEAD > dist/revision.txt')


@task
def update(ctx):
    # Create/populate a virtualenv that will be rsynced later along with the source.
    with ctx.lcd(settings.SRC_DIR):
        activate_script = os.path.join(settings.SRC_DIR, 'venv', 'bin', 'activate_this.py')
        # Peep doesn't yet cache downloaded files, so we reuse the virtualenv to speed up deploys.
        if not os.path.exists(activate_script):
            ctx.local('virtualenv --python=python2.7 venv')
        # Activate virtualenv.
        execfile(activate_script, dict(__file__=activate_script))
        # Install requirements using peep, so hashes are verified.
        with ctx.lcd(th_service_src):
            ctx.local('python2.7 bin/peep.py install -r requirements/common.txt')
        # Make the virtualenv relocatable since paths are hard-coded by default.
        ctx.local('virtualenv --relocatable venv')
        # Fix lib64 symlink to be relative instead of absolute.
        with ctx.lcd('venv'):
            ctx.local('rm -f lib64')
            ctx.local('ln -s lib lib64')

    with ctx.lcd(th_service_src):
        # Once we no longer use credentials.json, everything below apart from
        # collectstatic should be performed on one of the stage/prod specific
        # nodes at the end of the deploy, rather than on the admin node.

        # Generate gzipped versions of files that would benefit from compression, that
        # WhiteNoise can then serve in preference to the originals. This is required
        # since WhiteNoise's Django storage backend only gzips assets handled by
        # collectstatic, and so does not affect files in the `dist/` directory.
        ctx.local("python2.7 -m whitenoise.gzip dist")
        # Collect the static files (eg for the Persona or Django admin UI)
        run_local_with_env(ctx, "python2.7 manage.py collectstatic --noinput")
        # Update the database schema, if necessary.
        run_local_with_env(ctx, "python2.7 manage.py migrate --noinput")
        # Update reference data & tasks config from the in-repo fixtures.
        run_local_with_env(ctx, "python2.7 manage.py load_initial_data")
        # Populate the datasource table and create the connected databases.
        run_local_with_env(ctx, "python2.7 manage.py init_datasources")
        # Update oauth credentials.
        run_local_with_env(ctx, "python2.7 manage.py export_project_credentials")
        # Clear the cache.
        run_local_with_env(ctx, "python2.7 manage.py clear_cache")


@task
def deploy(ctx):
    # Use the local, IT-written deploy script to check in changes.
    ctx.local(settings.DEPLOY_SCRIPT)
    # Rsync the updated code to the nodes & restart processes. These are
    # separated out into their own functions, since the IRC bot output includes<commit_msg>Bug 1205758 - Run `npm install --production` during stage/prod deploy<commit_after>"""
Deploy this project in stage/production.

Requires commander_ which is installed on the systems that need it.

.. _commander: https://github.com/oremj/commander
"""

import os
import requests
import sys

from commander.deploy import hostgroups, task

sys.path.append(os.path.dirname(os.path.abspath(__file__)))
import commander_settings as settings  # noqa

env_file = os.path.join(settings.SRC_DIR, 'treeherder-env.sh')
th_service_src = os.path.join(settings.SRC_DIR, 'treeherder-service')

is_prod = 'treeherder.mozilla.org' in settings.SRC_DIR


def run_local_with_env(ctx, cmd):
    # For commands run from the admin node, we have to manually set the environment
    # variables, since the admin node is shared by both stage and prod.
    ctx.local("source {} && {}".format(env_file, cmd))


@task
def pre_update(ctx, ref=settings.UPDATE_REF):
    # Update the code to a specific git reference (branch/tag/sha) and write
    # info about the current repository state to a publicly visible file.
    with ctx.lcd(th_service_src):
        ctx.local('git fetch --quiet origin %s' % ref)
        ctx.local('git reset --hard FETCH_HEAD')
        # Fail early if we've forgotten to run grunt build before deploy.
        # This is easier to do (and more necessary) - now that dist/ is not
        # present on master, and only created on the stage/production branches.
        if not os.path.isfile(os.path.join(th_service_src, 'dist', 'index.html')):
            raise Exception("Grunt build hasn't been run and committed to the repo!")
        ctx.local('find . -type f -name "*.pyc" -delete')
        # Remove gzipped UI assets in case the uncompressed original no longer exists.
        ctx.local('find dist/ -type f -name "*.gz" -delete')
        ctx.local('git status -s')
        # Make the current Git revision accessible at <site-root>/revision.txt
        ctx.local('git rev-parse HEAD > dist/revision.txt')


@task
def update(ctx):
    # Create/populate a virtualenv that will be rsynced later along with the source.
    with ctx.lcd(settings.SRC_DIR):
        activate_script = os.path.join(settings.SRC_DIR, 'venv', 'bin', 'activate_this.py')
        # Peep doesn't yet cache downloaded files, so we reuse the virtualenv to speed up deploys.
        if not os.path.exists(activate_script):
            ctx.local('virtualenv --python=python2.7 venv')
        # Activate virtualenv.
        execfile(activate_script, dict(__file__=activate_script))
        # Install requirements using peep, so hashes are verified.
        with ctx.lcd(th_service_src):
            ctx.local('python2.7 bin/peep.py install -r requirements/common.txt')
        # Make the virtualenv relocatable since paths are hard-coded by default.
        ctx.local('virtualenv --relocatable venv')
        # Fix lib64 symlink to be relative instead of absolute.
        with ctx.lcd('venv'):
            ctx.local('rm -f lib64')
            ctx.local('ln -s lib lib64')

    with ctx.lcd(th_service_src):
        # Once we no longer use credentials.json, everything below apart from
        # collectstatic should be performed on one of the stage/prod specific
        # nodes at the end of the deploy, rather than on the admin node.

        # Install nodejs non-dev packages, needed for the grunt build.
        ctx.local("npm install --production")
        # Generate gzipped versions of files that would benefit from compression, that
        # WhiteNoise can then serve in preference to the originals. This is required
        # since WhiteNoise's Django storage backend only gzips assets handled by
        # collectstatic, and so does not affect files in the `dist/` directory.
        ctx.local("python2.7 -m whitenoise.gzip dist")
        # Collect the static files (eg for the Persona or Django admin UI)
        run_local_with_env(ctx, "python2.7 manage.py collectstatic --noinput")
        # Update the database schema, if necessary.
        run_local_with_env(ctx, "python2.7 manage.py migrate --noinput")
        # Update reference data & tasks config from the in-repo fixtures.
        run_local_with_env(ctx, "python2.7 manage.py load_initial_data")
        # Populate the datasource table and create the connected databases.
        run_local_with_env(ctx, "python2.7 manage.py init_datasources")
        # Update oauth credentials.
        run_local_with_env(ctx, "python2.7 manage.py export_project_credentials")
        # Clear the cache.
        run_local_with_env(ctx, "python2.7 manage.py clear_cache")


@task
def deploy(ctx):
    # Use the local, IT-written deploy script to check in changes.
    ctx.local(settings.DEPLOY_SCRIPT)
    # Rsync the updated code to the nodes & restart processes. These are
    # separated out into their own functions, since the IRC bot output includes
{'fraction_non_alphanumeric': 0.08887728459530025, 'fraction_numerical': 0.005744125326370757, 'mean_word_length': 3.0610687022900764, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7191436', 'n_tokens_mistral': 2782, 'n_tokens_neox': 2713, 'n_words': 993}