<commit_before>"""
Server layout:
    ~/services/
        This contains two subfolders
            /apache/
            /supervisor/
        which hold the configurations for these applications
        for each environment (staging, demo, etc) running on the server.
        Theses folders are included in the global /etc/apache2 and
        /etc/supervisor configurations.

    ~/www/
        This folder contains the code, python environment, and logs
        for each environment (staging, demo, etc) running on the server.
        Each environment has its own subfolder named for its evironment
        (i.e. ~/www/staging/logs and ~/www/demo/logs).
"""
import uuid
from fabric.context_managers import settings, cd
from fabric.operations import require, local, prompt

import os, sys

from fabric.api import run, roles, execute, task, sudo, env, parallel
from fabric.contrib import files, console
from fabric import utils
import posixpath
from collections import defaultdict

PROJECT_ROOT = os.path.dirname(__file__)
RSYNC_EXCLUDE = (
    '.DS_Store',
    '.git',
    '*.pyc',
    '*.example',
    '*.db',
    )
env.project = 'commcare-hq'
env.code_repo = 'git://github.com/dimagi/commcare-hq.git'
env.code_branch = 'master'
env.home = "/home/cchq"
env.selenium_url = 'http://jenkins.dimagi.com/job/commcare-hq-post-deploy/buildWithParameters?token=%(token)s&TARGET=%(environment)s'

env.roledefs = {
        'django_celery': [],
        'django_app': [],
        'django_public': [],
        'django_pillowtop': [], #for now combined with celery

        'django_monolith': [], # all of the above config - use this ONLY for single server config, lest deploy() will run multiple times in parallel causing bad contentions

        'formsplayer': [],
        'staticfiles': [],
        'remote_es': [], #remote elasticsearch ssh tunnel config

        #package level configs that are not quite config'ed yet in this fabfile
        'couch': [],
        'pg': [],
        'rabbitmq': [],
        'lb': [],

        'deploy': [], #a placeholder to ensure deploy only runs once on a bogus, non functioning task, to split out the real tasks in the execute() block

    }

@task
def _setup_path():
    # using posixpath to ensure unix style slashes. See bug-ticket: http://code.fabfile.org/attachments/61/posixpath.patch
    env.root = posixpath.join(env.home, 'www', env.environment)
    env.log_dir = posixpath.join(env.home, 'www', env.environment, 'log')
    env.code_root = posixpath.join(env.root, 'code_root')
    env.code_root_preindex = posixpath.join(env.root, 'code_root_preindex')
    env.project_root = posixpath.join(env.code_root, env.project)
    env.project_media = posixpath.join(env.code_root, 'media')
    env.virtualenv_root = posixpath.join(env.root, 'python_env')
    env.virtualenv_root_preindex = posixpath.join(env.root, 'python_env_preindex')
    env.services = posixpath.join(env.home, 'services')
    env.jython_home = '/usr/local/lib/jython'
    env.db = '%s_%s' % (env.project, env.environment)

@task
def _set_apache_user():
    if what_os() == 'ubuntu':
        env.apache_user = 'www-data'
    elif what_os() == 'redhat':
        env.apache_user = 'apache'

@roles('lb')
def setup_apache_dirs():
    sudo('mkdir -p %(services)s/apache' % env, user=env.sudo_user)

@roles('django_celery', 'django_app', 'staticfiles') #'django_public','formsplayer','staticfiles'
def setup_dirs():
    """ create (if necessary) and make writable uploaded media, log, etc. directories """
    sudo('mkdir -p %(log_dir)s' % env, user=env.sudo_user)
    sudo('chmod a+w %(log_dir)s' % env, user=env.sudo_user)
    sudo('mkdir -p %(services)s/supervisor' % env, user=env.sudo_user)
    #execute(setup_apache_dirs)

@task
def staging():
    """ use staging environment on remote host"""
    env.code_branch = 'develop'
    env.sudo_user = 'commcare-hq'
    env.environment = 'staging'
    env.django_port = '9002'
    env.server_name = 'noneset'
    env.hosts = ['192.168.56.1']
    env.settings = '%(project)s.localsettings' % env
    env.host_os_map = None
    _setup_path()
    env.user = prompt("Username: ", default='dimagivm')
    env.es_endpoint = 'localhost'

@task
def india():
    """Our production server in India."""
    env.home = '/home/commcarehq/'
    env.environment = 'india'
    env.sudo_user = 'commcarehq'
    env.hosts = ['172.16.58.3']
    env.user = prompt("Username: ", default=env.user)
    env.django_port = '8001'

    _setup_path()
    env.virtualenv_root = posixpath.join(env.home, '.virtualenvs/commcarehq')
    env.virtualenv_root_preindex = posixpath.join(env.home, '.virtualenvs/commcarehq_preindex')

    env.roledefs = {
        'couch': [],
        'pg': [],
        'rabbitmq': [],
        'sofabed': [],
        'django_celery': [],
        'django_app': [],
        'django_public': [],
        'django_pillowtop': [],
        'formsplayer': [],
        'remote_es': [],
        'staticfiles': [],
        'lb': [],
        'deploy': [],

        'django_monolith': ['172.16.58.3'],
    }
    env.roles = ['django_monolith']
    env.es_endpoint = 'localhost'

@task
def production():
    """ use production environment on remote host"""
    env.sudo_user = 'cchq'
    env.environment = 'production'
    env.django_port = '9010'

    #env.hosts = None
    env.roledefs = {
        'couch': ['hqdb.internal.commcarehq.org'],
        'pg': ['hqdb.internal.commcarehq.org'],
        'rabbitmq': ['hqdb.internal.commcarehq.org'],
        'sofabed': ['hqdb.internal.commcarehq.org'], #todo, right now group it with celery
        'django_celery': ['hqdb.internal.commcarehq.org'],
        'django_app': ['hqdjango0.internal.commcarehq.org', 'hqdjango2.internal.commcarehq.org'],
        'django_public': ['hqdjango1.internal.commcarehq.org',],
        'django_pillowtop': ['hqdb.internal.commcarehq.org'],

        'remote_es': ['hqdb.internal.commcarehq.org', 'hqdjango0.internal.commcarehq.org',
                      'hqdjango1.internal.commcarehq.org', 'hqdjango2.internal.commcarehq.org'],

        'formsplayer': ['hqdjango0.internal.commcarehq.org'],
        'lb': [], #todo on apache level config
        'staticfiles': ['hqproxy0.internal.commcarehq.org'],
        'deploy': ['hqdb.internal.commcarehq.org'], #this is a stub becuaue we don't want to be prompted for a host or run deploy too many times
        'django_monolith': [] # fab complains if this doesn't exist
    }


    env.server_name = 'commcare-hq-production'
    env.settings = '%(project)s.localsettings' % env
    env.host_os_map = None # e.g. 'ubuntu' or 'redhat'.  Gets autopopulated by what_os() if you don't know what it is or don't want to specify.
    env.roles = ['deploy', ]
    env.es_endpoint = 'hqes0.internal.commcarehq.org'''

    _setup_path()

@task
def realstaging():
    """ use production environment on remote host"""
    env.code_branch = 'pact-dev'
    env.sudo_user = 'cchq'
    env.environment = 'staging'
    env.django_port = '9010'

    #env.hosts = None
    env.roledefs = {
        'couch': ['hqdb-staging.internal.commcarehq.org'],
        'pg': ['hqdb-staging.internal.commcarehq.org'],
        'rabbitmq': ['hqdb-staging.internal.commcarehq.org'],
        'sofabed': ['hqdb-staging.internal.commcarehq.org'], #todo, right now group it with celery<commit_msg>change realstaging to build off the master branch<commit_after>"""
Server layout:
    ~/services/
        This contains two subfolders
            /apache/
            /supervisor/
        which hold the configurations for these applications
        for each environment (staging, demo, etc) running on the server.
        Theses folders are included in the global /etc/apache2 and
        /etc/supervisor configurations.

    ~/www/
        This folder contains the code, python environment, and logs
        for each environment (staging, demo, etc) running on the server.
        Each environment has its own subfolder named for its evironment
        (i.e. ~/www/staging/logs and ~/www/demo/logs).
"""
import uuid
from fabric.context_managers import settings, cd
from fabric.operations import require, local, prompt

import os, sys

from fabric.api import run, roles, execute, task, sudo, env, parallel
from fabric.contrib import files, console
from fabric import utils
import posixpath
from collections import defaultdict

PROJECT_ROOT = os.path.dirname(__file__)
RSYNC_EXCLUDE = (
    '.DS_Store',
    '.git',
    '*.pyc',
    '*.example',
    '*.db',
    )
env.project = 'commcare-hq'
env.code_repo = 'git://github.com/dimagi/commcare-hq.git'
env.code_branch = 'master'
env.home = "/home/cchq"
env.selenium_url = 'http://jenkins.dimagi.com/job/commcare-hq-post-deploy/buildWithParameters?token=%(token)s&TARGET=%(environment)s'

env.roledefs = {
        'django_celery': [],
        'django_app': [],
        'django_public': [],
        'django_pillowtop': [], #for now combined with celery

        'django_monolith': [], # all of the above config - use this ONLY for single server config, lest deploy() will run multiple times in parallel causing bad contentions

        'formsplayer': [],
        'staticfiles': [],
        'remote_es': [], #remote elasticsearch ssh tunnel config

        #package level configs that are not quite config'ed yet in this fabfile
        'couch': [],
        'pg': [],
        'rabbitmq': [],
        'lb': [],

        'deploy': [], #a placeholder to ensure deploy only runs once on a bogus, non functioning task, to split out the real tasks in the execute() block

    }

@task
def _setup_path():
    # using posixpath to ensure unix style slashes. See bug-ticket: http://code.fabfile.org/attachments/61/posixpath.patch
    env.root = posixpath.join(env.home, 'www', env.environment)
    env.log_dir = posixpath.join(env.home, 'www', env.environment, 'log')
    env.code_root = posixpath.join(env.root, 'code_root')
    env.code_root_preindex = posixpath.join(env.root, 'code_root_preindex')
    env.project_root = posixpath.join(env.code_root, env.project)
    env.project_media = posixpath.join(env.code_root, 'media')
    env.virtualenv_root = posixpath.join(env.root, 'python_env')
    env.virtualenv_root_preindex = posixpath.join(env.root, 'python_env_preindex')
    env.services = posixpath.join(env.home, 'services')
    env.jython_home = '/usr/local/lib/jython'
    env.db = '%s_%s' % (env.project, env.environment)

@task
def _set_apache_user():
    if what_os() == 'ubuntu':
        env.apache_user = 'www-data'
    elif what_os() == 'redhat':
        env.apache_user = 'apache'

@roles('lb')
def setup_apache_dirs():
    sudo('mkdir -p %(services)s/apache' % env, user=env.sudo_user)

@roles('django_celery', 'django_app', 'staticfiles') #'django_public','formsplayer','staticfiles'
def setup_dirs():
    """ create (if necessary) and make writable uploaded media, log, etc. directories """
    sudo('mkdir -p %(log_dir)s' % env, user=env.sudo_user)
    sudo('chmod a+w %(log_dir)s' % env, user=env.sudo_user)
    sudo('mkdir -p %(services)s/supervisor' % env, user=env.sudo_user)
    #execute(setup_apache_dirs)

@task
def staging():
    """ use staging environment on remote host"""
    env.code_branch = 'develop'
    env.sudo_user = 'commcare-hq'
    env.environment = 'staging'
    env.django_port = '9002'
    env.server_name = 'noneset'
    env.hosts = ['192.168.56.1']
    env.settings = '%(project)s.localsettings' % env
    env.host_os_map = None
    _setup_path()
    env.user = prompt("Username: ", default='dimagivm')
    env.es_endpoint = 'localhost'

@task
def india():
    """Our production server in India."""
    env.home = '/home/commcarehq/'
    env.environment = 'india'
    env.sudo_user = 'commcarehq'
    env.hosts = ['172.16.58.3']
    env.user = prompt("Username: ", default=env.user)
    env.django_port = '8001'

    _setup_path()
    env.virtualenv_root = posixpath.join(env.home, '.virtualenvs/commcarehq')
    env.virtualenv_root_preindex = posixpath.join(env.home, '.virtualenvs/commcarehq_preindex')

    env.roledefs = {
        'couch': [],
        'pg': [],
        'rabbitmq': [],
        'sofabed': [],
        'django_celery': [],
        'django_app': [],
        'django_public': [],
        'django_pillowtop': [],
        'formsplayer': [],
        'remote_es': [],
        'staticfiles': [],
        'lb': [],
        'deploy': [],

        'django_monolith': ['172.16.58.3'],
    }
    env.roles = ['django_monolith']
    env.es_endpoint = 'localhost'

@task
def production():
    """ use production environment on remote host"""
    env.sudo_user = 'cchq'
    env.environment = 'production'
    env.django_port = '9010'

    #env.hosts = None
    env.roledefs = {
        'couch': ['hqdb.internal.commcarehq.org'],
        'pg': ['hqdb.internal.commcarehq.org'],
        'rabbitmq': ['hqdb.internal.commcarehq.org'],
        'sofabed': ['hqdb.internal.commcarehq.org'], #todo, right now group it with celery
        'django_celery': ['hqdb.internal.commcarehq.org'],
        'django_app': ['hqdjango0.internal.commcarehq.org', 'hqdjango2.internal.commcarehq.org'],
        'django_public': ['hqdjango1.internal.commcarehq.org',],
        'django_pillowtop': ['hqdb.internal.commcarehq.org'],

        'remote_es': ['hqdb.internal.commcarehq.org', 'hqdjango0.internal.commcarehq.org',
                      'hqdjango1.internal.commcarehq.org', 'hqdjango2.internal.commcarehq.org'],

        'formsplayer': ['hqdjango0.internal.commcarehq.org'],
        'lb': [], #todo on apache level config
        'staticfiles': ['hqproxy0.internal.commcarehq.org'],
        'deploy': ['hqdb.internal.commcarehq.org'], #this is a stub becuaue we don't want to be prompted for a host or run deploy too many times
        'django_monolith': [] # fab complains if this doesn't exist
    }


    env.server_name = 'commcare-hq-production'
    env.settings = '%(project)s.localsettings' % env
    env.host_os_map = None # e.g. 'ubuntu' or 'redhat'.  Gets autopopulated by what_os() if you don't know what it is or don't want to specify.
    env.roles = ['deploy', ]
    env.es_endpoint = 'hqes0.internal.commcarehq.org'''

    _setup_path()

@task
def realstaging():
    """ use production environment on remote host"""
    env.code_branch = 'master'
    env.sudo_user = 'cchq'
    env.environment = 'staging'
    env.django_port = '9010'

    #env.hosts = None
    env.roledefs = {
        'couch': ['hqdb-staging.internal.commcarehq.org'],
        'pg': ['hqdb-staging.internal.commcarehq.org'],
        'rabbitmq': ['hqdb-staging.internal.commcarehq.org'],
        'sofabed': ['hqdb-staging.internal.commcarehq.org'], #todo, right now group it with celery
{'fraction_non_alphanumeric': 0.1458703071672355, 'fraction_numerical': 0.007235494880546075, 'mean_word_length': 3.1610337972167, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 12, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6379590', 'n_tokens_mistral': 5059, 'n_tokens_neox': 4882, 'n_words': 1106}