<commit_before>import os
import re
import sys
from getpass import getpass, getuser
from glob import glob
from contextlib import contextmanager
from posixpath import join
import tempfile
from StringIO import StringIO
import traceback
from time import sleep
from fabric.context_managers import settings
from fabric.api import env, cd, run as _run, hide, task, get, puts, put, roles, execute, parallel
from fabric.contrib.console import confirm
from fabric.contrib.files import exists, upload_template, _escape_for_regex, append
from fabric.colors import red
from fabric.utils import *
import simplejson
from utils import log_call, pip, print_command, sudo, project

################
# Config setup #
################

# removes the port


def get_host(host_name):
    if host_name.find(":") != -1:
        return host_name[0:host_name.find(":")]
    else:
        return host_name


def load_environment(conf, show_info):
    if show_info:
        print simplejson.dumps(conf, sort_keys=True, indent=4)

    if not "APPLICATION_HOSTS" in conf or len(conf["APPLICATION_HOSTS"]) == 0:
        abort("No application hosts are defined")
    if not "DATABASE_HOSTS" in conf or len(conf["DATABASE_HOSTS"]) == 0:
        abort("No database hosts are defined")

    env.db_pass = conf.get("DB_PASS", None)
    env.admin_pass = conf.get("ADMIN_PASS", None)
    env.user = conf.get("SSH_USER", getuser())
    env.password = conf.get("SSH_PASS", None)
    env.key_filename = conf.get("SSH_KEY_PATH", None)
    env.roledefs = {
        'application': conf.get("APPLICATION_HOSTS"),
        'database': conf.get("DATABASE_HOSTS")[0:1],
        'db_slave': conf.get("DATABASE_HOSTS")[1:],
        'cron': conf.get("CRON_HOSTS", conf.get("APPLICATION_HOSTS")),
    }
    env.application_hosts = conf.get("APPLICATION_HOSTS") # used for our own consumption
    env.cron_hosts = conf.get("CRON_HOSTS", conf.get("APPLICATION_HOSTS")) # used for our own consumption
    env.database_hosts = conf.get("DATABASE_HOSTS") # used for matching public and private database host IP addresses
    env.private_database_hosts = conf.get("PRIVATE_DATABASE_HOSTS", ["127.0.0.1"])
    env.primary_database_host = env.private_database_hosts[0] # the first listed private database host is the master, used by live_settings.py
    env.private_application_hosts = conf.get("PRIVATE_APPLICATION_HOSTS", ["127.0.0.1"])
    env.private_cron_hosts = conf.get("PRIVATE_CRON_HOSTS", env.private_application_hosts)
    if conf.get("LIVE_HOSTNAME"):
        tmp_hosts = list()
        tmp_hosts.extend(conf.get("APPLICATION_HOSTS"))
        if conf.get("LIVE_HOSTNAME") and not conf.get("LIVE_HOSTNAME") in tmp_hosts: # don't duplicate any host names
            tmp_hosts.append(conf.get("LIVE_HOSTNAME"))
        if not "127.0.0.1" in tmp_hosts: # nginx always comes from local host
            tmp_hosts.append("127.0.0.1")

        env.allowed_hosts = ",".join(["'%s'" % get_host(host) for host in tmp_hosts]) # used by live_settings.py to set Django's allowed hosts
    else:
        env.allowed_hosts = ",".join(["'%s'" % get_host(host) for host in conf.get("APPLICATION_HOSTS")]) # used by live_settings.py to set Django's allowed hosts

    assert len(env.private_database_hosts) == len(env.database_hosts), "Same number of DATABASE_HOSTS and PRIVATE_DATABASE_HOSTS must be listed"
    assert len(env.private_application_hosts) == len(env.application_hosts), "Same number of APPLICATION_HOSTS and PRIVATE_APPLICATION_HOSTS must be listed"
    assert len(env.private_cron_hosts) == len(env.cron_hosts), "Same number of CRON_HOSTS and PRIVATE_CRON_HOSTS must be listed"

    env.proj_name = conf.get("PROJECT_NAME", os.getcwd().split(os.sep)[-1])
    env.venv_home = conf.get("VIRTUALENV_HOME", "/home/%s" % env.user)
    env.venv_path = "%s/%s" % (env.venv_home, env.proj_name)
    env.proj_dirname = "project"
    env.proj_path = conf.get("PROJECT_PATH", "%s/%s" % (env.venv_path, env.proj_dirname))
    env.manage = "%s/bin/python %s/project/manage.py" % (env.venv_path,
                                                         env.venv_path)
    env.live_host = conf.get("LIVE_HOSTNAME", get_host(conf.get("APPLICATION_HOSTS")[0]) if conf.get("APPLICATION_HOSTS") else None)
    env.sitename = conf.get("SITENAME", "Default")
    env.repo_url = conf.get("REPO_URL", "")
    env.git = True
    env.reqs_path = conf.get("REQUIREMENTS_PATH", None)
    env.gunicorn_port = conf.get("GUNICORN_PORT", 8000)
    env.locale = conf.get("LOCALE", "en_US.UTF-8")
    env.linux_distro = conf.get("LINUX_DISTRO", "squeeze")

    env.deploy_my_public_key = conf.get("DEPLOY_MY_PUBLIC_KEY")
    env.deploy_ssh_key_path = conf.get("DEPLOY_SSH_KEY_PATH")
    env.deploy_db_cluster_key_path = conf.get("DEPLOY_DB_CLUSTER_SSH_KEY_PATH")
    env.apt_requirements = conf.get("APT_REQUIREMENTS", [])


if sys.argv[0].split(os.sep)[-1] in ("fab",             # POSIX
                                     "fab-script.py"):  # Windows
    # Ensure we import settings from the current dir
    try:
        global_conf = {}
        global_conf = __import__("fabric_settings", globals(), locals(), [], 0).FABRIC
        try:
            env.settings = global_conf # save settings to switch later
            env.mode = "development"
            load_environment(env.settings[env.mode], False)

        except (KeyError, ValueError):
            raise ImportError
    except (ImportError, AttributeError):
        if not confirm("Warning, no hosts defined: Are you sure you want to continue?"):
            print "\nAborting!"
            exit()


##################
# Template setup #
##################

# Each template gets uploaded at deploy time, only if their
# contents has changed, in which case, the reload command is
# also run.
#
# If a role entry is specified, the template is uploaded only for the given role: application, cron

templates = {
    "nginx": {
        "local_path": "deploy/nginx.conf",
        "remote_path": "/etc/nginx/sites-enabled/%(proj_name)s.conf",
        "reload_command": "service nginx restart",
    },
    "supervisor": {
        "local_path": "deploy/supervisor.conf",
        "remote_path": "/etc/supervisor/conf.d/%(proj_name)s.conf",
        "reload_command": "supervisorctl reload",
    },
    "cron": {
        "local_path": "deploy/crontab",
        "remote_path": "/etc/cron.d/%(proj_name)s",
        "owner": "root",
        "mode": "600",
        "role": "cron",
    },
    "gunicorn": {
        "local_path": "deploy/gunicorn.conf.py",
        "remote_path": "%(proj_path)s/gunicorn.conf.py",
    },
    "settings": {
        "local_path": "deploy/%(mode)s_settings.py", # local task changes this filename
        "remote_path": "%(proj_path)s/local_settings.py",
    },
    "celery": {
        "local_path": "deploy/celeryd.conf",
        "remote_path": "/etc/supervisor/conf.d/%(proj_name)s.celery.conf",
        "required_module": "celery",
        "reload_command": "supervisorctl reload",
    },
}

# Additional templates are created in code.
#
#  ssh
#       ~/.ssh/authorized_keys
#       ~/.ssh/config
#       ~/.ssh/%(DEPLOY_SSH_KEY_PATH.basename())s
#
#  ssh_db
#       ~postgres/.ssh/known_hosts
#       ~postgres/.ssh/config
#       ~postgres/.ssh/authorized_keys
#       ~postgres/.ssh/$(DEPLOY_DB_CLUSTER_SSH_KEY_PATH.basename())s
#
#  postgresql
#      /etc/postgresql/9.2/main/postgresql.conf
#      /etc/postgresql/9.2/main/pg_hba.conf
#
#  db_slave
#      /var/lib/postgresql/9.2/main/recovery.conf
#      /var/lib/postgresql/9.2/archive
#
#  database, db_slave
#      /var/tmp/pg_basebackup_%(proj_name)s.tar.bz2


####################
#  Utility Methods #
####################

CONFIG_FILE_NORMAL = "normal"
CONFIG_FILE_RECORDS = "records"

def modify_config_file(remote_path, settings=None, comment_char='#', setter_char='=', type=CONFIG_FILE_NORMAL,
    use_sudo=False, backup=True):

    """
    Alter the settings of an existing remote config file, trying to uncomment found settings, adding them otherwise.

    The config file is loaded to a local temporary file, processed, and uploaded back to the ``remote_path``.

    By default, backup is made remotely. Set ``backup`` to False to prevent this behavior.

    Two kinds of config files are currently supported.
    Default config file ``type`` is 'normal' where lines look like this:

        # - Connection Settings -
        listen_addresses = 'localhost' # what IP address(es) to listen on;
        #port = 5432

    The ``settings`` must be a list of tuples :

        settings = [('listen_addresses', "'*'"), ('port', '5433')]

    The script will change the config file to :

        # - Connection Settings -
        listen_addresses = '*' # what IP address(es) to listen on;
        port = 5433

    Notice how the line with the port setting was uncommented and set to a new value.
    If an existing line is not found, it is added.

    The ``comment_char`` defaults to '#' and ``setter_char``defaults to '='.

    A basic 'records' config file ``type`` is also supported. Ex: an pg_hba.conf file :

    # TYPE  DATABASE        USER            ADDRESS                 METHOD

    # "local" is for Unix domain socket connections only
    local   all             all                                     peer
    # IPv4 local connections:
    host    all             all             127.0.0.1/32            md5
    # IPv6 local connections:
    host    all             all             ::1/128                 md5
    # Allow replication connections from localhost, by a user with the
    # replication privilege.
    #local   replication     postgres                                peer
    #host    replication     postgres        127.0.0.1/32            md5
    #host    replication     postgres        ::1/128                 md5


    In such a file, we would change the IPv4 local connection to 'trust' and enable local replication like this :
    modify_config_file('/etc/postgresql/9.1/main/pg_hba.conf',
                [('host', 'all', 'all', '127.0.0.1/32','trust'),
                ('local', 'replication', '', 'peer')], type="records")

    By default, the file will be copied to ``remote_path`` as the logged-in
    user; specify ``use_sudo=True`` to use `sudo` instead.

    Downloaded from https://github.com/fabric/fabric/pull/658

    """
    changes_done = set()
    options = dict(comment_char=comment_char, setter_char=setter_char)

    # Create a temporary file
    with tempfile.NamedTemporaryFile(delete=True) as f:

        # Download the remote file into the temporary file
        a = get(remote_path, f)

        # Rewind the file to the beginning
        f.file.seek(0)

        # We're going to read each line and put them into newlines
        newlines = list()

        for line in f.file.readlines():

            # Compare each line with the wanted changes
            for setting in settings:
                # Check for a possible mistake of the user
                if not isinstance(setting, list) and not isinstance(setting, tuple):
                    tb = traceback.format_exc()
                    abort(tb + "\nChanges must be a tuple/list of tuples/lists.\n%s is not a list nor a tuple." % (setting,))

                # Case of a normal config file : each line is of the kind some_variable = some_value
                if type == CONFIG_FILE_NORMAL:
                    key, value = setting

                    # Search for a line of the form : some_variable = some_value
                    # or commented: #some_variable = some_value
                    # We preserve the ending \n.
                    search = ("\s*(?P<is_commented>%(comment_char)s*)(?P<left_chunk>\s*"+_escape_for_regex(key)
                              +"\s*%(setter_char)s\s*)(?P<right_chunk>\s*.*\n*)") % options
                    m = re.match(search, line)

                    if m:
                        # A match was found. Check for a comment at the end of the line,
                        # like some_variable = some_value # this is some comment
                        r = m.groupdict()
                        right_chunk = r['right_chunk']
                        comment = "\n"
                        if right_chunk is not None:
                            # We will capture the comment at the end of the line, if any
                            m2 = re.match("(?P<old_value>.*?)(?P<comment>\s*%(comment_char)s+.*\n*)" % options, right_chunk)
                            if m2:
                                comment = m2.groupdict()['comment']

                        puts('Found line:     ' + line, end='')

                        # It's not rare to find multiple commented versions of a setting in a config file.
                        # If the change was applied in the loop before, make sure we don't set the variable twice in
                        # this case.
                        if setting in changes_done:
                            if r['is_commented']:
                                # This is a commented setting, leave it so.
                                puts('Left it commented.')
                            else:
                                # This is a setting that's not commented, but we set a similar setting earlier in the
                                # loop either because we found a commented version and uncommented it then  applied the
                                # new value, or because it was a duplicate.
                                # In any case, we comment this new setting so our first setting takes precedence.
                                puts('Commented it:   '+red(comment_char) + line)
                                # Recreate the line using the new value set by the user
                                line = comment_char + line
                        else:
                            # Recreate the line using the new value set by the user
                            line = r['left_chunk'] + str(value) + comment

                            puts('Replaced with:  '+r['left_chunk'] + red(value) + comment)

                        # Mark the change as done
                        changes_done.add(setting)

                elif type == CONFIG_FILE_RECORDS:
                    escaped_change = [_escape_for_regex(t) for t in setting]
                    search = ("\s*(?P<is_commented>%(comment_char)s*)(?P<first_values>\s*"
                              + "(\s+)".join(escaped_change[:-1]+[')(?P<right_chunk>.*\n*)'])) % options

                    m = re.match(search, line)
                    if m:
                        r = m.groupdict()
                        right_chunk = r['right_chunk']
                        comment = "\n"
                        if right_chunk is not None:
                            # We will capture the comment at the end of the line, if any
                            m2 = re.match(".*?(?P<comment>\s*%(comment_char)s+.*\n*)" % options, right_chunk)
                            if m2:
                                comment = m2.groupdict()['comment']

                        puts('Found line:     ' + line, end='')

                        if setting in changes_done:
                            if r['is_commented']:
                                puts('Left it commented')
                            else:
                                puts('Commented it:   '+red(comment_char) + line)
                                # Recreate the line using the new value set by the user
                                line = comment_char + line
                        else:
                            # Recreate the line using the new value set by the user
                            line = r['first_values'] + str(setting[-1]) + comment
                            puts('Replaced with:  '+ r['first_values'] + red(str(setting[-1])) + comment)


                        changes_done.add(setting)

            # append the line
            newlines.append(line)

        # So far, some changes have been satisfied : those that were possible by removing the comment on existing lines
        # and setting a new value.
        # Let's now add new lines for remaining changes.
        for setting in settings:
            if not setting in changes_done:
                if type == CONFIG_FILE_NORMAL:
                    line = "%s %s %s\n" % (setting[0], setter_char, setting[1])
                    print line
                    newlines.append(line)
                elif type == CONFIG_FILE_RECORDS:
                    line = "\t".join(setting) + "\n"
                    print line
                    newlines.append(line)

        func = use_sudo and sudo or run

        if backup:
            func("cp %s{,.bak}" % remote_path)

        newlines.append("")

        with tempfile.NamedTemporaryFile(delete=True) as new_file:
            new_file.writelines(newlines)

            # Upload the file.
            put(
                local_path=new_file,
                remote_path=remote_path,
                use_sudo=use_sudo
                #mirror_local_mode=mirror_local_mode,
                #mode=mode
            )


def append_to_remote_file_top(remote_filename, line):
    """
    Replaces the given line with the same at the top of the file.
    """
    if not exists(remote_filename):
        append(remote_filename, line)
    else:
        # read the file and look for the string not using regex
        test = run("cat %s" % remote_filename, show=False)
        if test.find(line) != -1:
            # this code moves the following line to the top of the file
            file = StringIO()
            get(remote_filename, file)
            new_file = [line]
            new_file.extend([l for l in file.getvalue().split("\n") if l != line])
            new_text = "\n".join(new_file) + "\n"
            put(StringIO(new_text), remote_filename)
        else:
            append(remote_filename, line)


def fancy_append(remote_filename, line):
    """
    Fixes append() because it duplicates content of authorized_keys. Returns False if no operation, True if a file was appended.
    """
    if not exists(remote_filename):
        append(remote_filename, line)
    else:
        # read the file and look for the string not using regex
        test = run("cat %s" % remote_filename, show=False)
        if test.find(line) != -1:
            return True
        else:
            append(remote_filename, line)


def check_requirements_file(module):
    """
    Checks a requirements file to see if a module is installed.
    """
    with open(env.reqs_path, "r") as f:
        return f.read().find(module) != -1

######################################
# Context for virtualenv and project #
######################################


@contextmanager
def update_changed_requirements():
    """
    Checks for changes in the requirements file across an update,
    and gets new requirements if changes have occurred.
    """
    reqs_path = join(env.proj_path, env.reqs_path)
    get_reqs = lambda: run("cat %s" % reqs_path, show=False)
    old_reqs = get_reqs() if env.reqs_path else ""
    yield
    if old_reqs:
        new_reqs = get_reqs()
        if old_reqs == new_reqs:
            # Unpinned requirements should always be checked.
            for req in new_reqs.split("\n"):
                if req.startswith("-e"):
                    if "@" not in req:
                        # Editable requirement without pinned commit.
                        break
                elif req.strip() and not req.startswith("#"):
                    if not set(">=<") & set(req):
                        # PyPI requirement without version.
                        break
            else:
                # All requirements are pinned.
                return
        pip("-r %s/%s" % (env.proj_path, env.reqs_path))


###########################################
# Utils and wrappers for various commands #
###########################################

def _print(output):
    print
    print output
    print


@task
@roles('application','cron','database','db_slave')
def run(command, show=True):
    """
    Runs a shell comand on the remote server.
    """
    if show:
        print_command(command)
    with hide("running"):
        return _run(command)


def get_templates():
    """
    Returns each of the templates with env vars injected.
    """
    injected = {}
    for name, data in templates.items():
        injected[name] = dict([(k, v % env) for k, v in data.items()])
    return injected


def upload_template_and_reload(name):
    """
    Uploads a template only if it has changed, and if so, reload a
    related service.
    """
    template = get_templates()[name]
    local_path = template["local_path"]
    remote_path = template["remote_path"]
    reload_command = template.get("reload_command")
    owner = template.get("owner")
    mode = template.get("mode")
    remote_data = ""
    required_module = template.get("required_module")

    # block uploading certain files (e.g., celery configuration) if it is not installed in requirements
    if required_module and not check_requirements_file(required_module):
        return

    if exists(remote_path):
        with hide("stdout"):
            remote_data = sudo("cat %s" % remote_path, show=False)
    print "local_path: %s " % local_path
    with open(local_path, "r") as f:
        local_data = f.read()
        # Escape all non-string-formatting-placeholder occurrences of '%':
        local_data = re.sub(r"%(?!\(\w+\)s)", "%%", local_data)
        if "%(db_pass)s" in local_data:
            env.db_pass = db_pass()
        local_data %= env
    clean = lambda s: s.replace("\n", "").replace("\r", "").strip()
    if clean(remote_data) == clean(local_data):
        return
    upload_template(local_path, remote_path, env, use_sudo=True, backup=False)
    if owner:
        sudo("chown %s %s" % (owner, remote_path))
    if mode:
        sudo("chmod %s %s" % (mode, remote_path))
    if reload_command:
        sudo(reload_command)


def db_pass():
    """
    Prompts for the database password if unknown.
    """
    if not env.db_pass:
        env.db_pass = getpass("Enter the database password: ")
    return env.db_pass


@task
@roles('application','cron','database','db_slave')
def apt(packages):
    """
    Installs one or more system packages via apt.
    """
    return sudo("apt-get install -y -q " + packages)


def postgres(command):
    """
    Runs the given command as the postgres user.
    """
    show = not command.startswith("psql")
    return run("sudo -u root sudo -u postgres %s" % command, show=show)


@task
@roles('database')
def psql(sql, show=True):
    """
    Runs SQL against the project's database.
    """
    out = postgres('psql -c "%s"' % sql)<commit_msg>check for invalid db passwords (passwords that break the script)<commit_after>import os
import re
import sys
from getpass import getpass, getuser
from glob import glob
from contextlib import contextmanager
from posixpath import join
import tempfile
from StringIO import StringIO
import traceback
from time import sleep
from fabric.context_managers import settings
from fabric.api import env, cd, run as _run, hide, task, get, puts, put, roles, execute, parallel
from fabric.contrib.console import confirm
from fabric.contrib.files import exists, upload_template, _escape_for_regex, append
from fabric.colors import red
from fabric.utils import *
import simplejson
from utils import log_call, pip, print_command, sudo, project

################
# Config setup #
################

# removes the port


# this function has to be defined before load_environment()
def check_db_password(password):
    chars = set('$()@') # characters to block
    return not any((c in chars) for c in password)


def get_host(host_name):
    if host_name.find(":") != -1:
        return host_name[0:host_name.find(":")]
    else:
        return host_name


def load_environment(conf, show_info):
    if show_info:
        print simplejson.dumps(conf, sort_keys=True, indent=4)

    if not "APPLICATION_HOSTS" in conf or len(conf["APPLICATION_HOSTS"]) == 0:
        abort("No application hosts are defined")
    if not "DATABASE_HOSTS" in conf or len(conf["DATABASE_HOSTS"]) == 0:
        abort("No database hosts are defined")

    env.db_pass = conf.get("DB_PASS", None)
    env.admin_pass = conf.get("ADMIN_PASS", None)
    env.user = conf.get("SSH_USER", getuser())
    env.password = conf.get("SSH_PASS", None)
    env.key_filename = conf.get("SSH_KEY_PATH", None)
    env.roledefs = {
        'application': conf.get("APPLICATION_HOSTS"),
        'database': conf.get("DATABASE_HOSTS")[0:1],
        'db_slave': conf.get("DATABASE_HOSTS")[1:],
        'cron': conf.get("CRON_HOSTS", conf.get("APPLICATION_HOSTS")),
    }
    env.application_hosts = conf.get("APPLICATION_HOSTS") # used for our own consumption
    env.cron_hosts = conf.get("CRON_HOSTS", conf.get("APPLICATION_HOSTS")) # used for our own consumption
    env.database_hosts = conf.get("DATABASE_HOSTS") # used for matching public and private database host IP addresses
    env.private_database_hosts = conf.get("PRIVATE_DATABASE_HOSTS", ["127.0.0.1"])
    env.primary_database_host = env.private_database_hosts[0] # the first listed private database host is the master, used by live_settings.py
    env.private_application_hosts = conf.get("PRIVATE_APPLICATION_HOSTS", ["127.0.0.1"])
    env.private_cron_hosts = conf.get("PRIVATE_CRON_HOSTS", env.private_application_hosts)
    if conf.get("LIVE_HOSTNAME"):
        tmp_hosts = list()
        tmp_hosts.extend(conf.get("APPLICATION_HOSTS"))
        if conf.get("LIVE_HOSTNAME") and not conf.get("LIVE_HOSTNAME") in tmp_hosts: # don't duplicate any host names
            tmp_hosts.append(conf.get("LIVE_HOSTNAME"))
        if not "127.0.0.1" in tmp_hosts: # nginx always comes from local host
            tmp_hosts.append("127.0.0.1")

        env.allowed_hosts = ",".join(["'%s'" % get_host(host) for host in tmp_hosts]) # used by live_settings.py to set Django's allowed hosts
    else:
        env.allowed_hosts = ",".join(["'%s'" % get_host(host) for host in conf.get("APPLICATION_HOSTS")]) # used by live_settings.py to set Django's allowed hosts

    assert len(env.private_database_hosts) == len(env.database_hosts), "Same number of DATABASE_HOSTS and PRIVATE_DATABASE_HOSTS must be listed"
    assert len(env.private_application_hosts) == len(env.application_hosts), "Same number of APPLICATION_HOSTS and PRIVATE_APPLICATION_HOSTS must be listed"
    assert len(env.private_cron_hosts) == len(env.cron_hosts), "Same number of CRON_HOSTS and PRIVATE_CRON_HOSTS must be listed"

    env.proj_name = conf.get("PROJECT_NAME", os.getcwd().split(os.sep)[-1])
    env.venv_home = conf.get("VIRTUALENV_HOME", "/home/%s" % env.user)
    env.venv_path = "%s/%s" % (env.venv_home, env.proj_name)
    env.proj_dirname = "project"
    env.proj_path = conf.get("PROJECT_PATH", "%s/%s" % (env.venv_path, env.proj_dirname))
    env.manage = "%s/bin/python %s/project/manage.py" % (env.venv_path,
                                                         env.venv_path)
    env.live_host = conf.get("LIVE_HOSTNAME", get_host(conf.get("APPLICATION_HOSTS")[0]) if conf.get("APPLICATION_HOSTS") else None)
    env.sitename = conf.get("SITENAME", "Default")
    env.repo_url = conf.get("REPO_URL", "")
    env.git = True
    env.reqs_path = conf.get("REQUIREMENTS_PATH", None)
    env.gunicorn_port = conf.get("GUNICORN_PORT", 8000)
    env.locale = conf.get("LOCALE", "en_US.UTF-8")
    env.linux_distro = conf.get("LINUX_DISTRO", "squeeze")

    env.deploy_my_public_key = conf.get("DEPLOY_MY_PUBLIC_KEY")
    env.deploy_ssh_key_path = conf.get("DEPLOY_SSH_KEY_PATH")
    env.deploy_db_cluster_key_path = conf.get("DEPLOY_DB_CLUSTER_SSH_KEY_PATH")
    env.apt_requirements = conf.get("APT_REQUIREMENTS", [])


    # safety check the db password
    if not check_db_password(env.db_pass):
        abort("The database password contains disallowed special characters.")

if sys.argv[0].split(os.sep)[-1] in ("fab",             # POSIX
                                     "fab-script.py"):  # Windows
    # Ensure we import settings from the current dir
    try:
        global_conf = {}
        global_conf = __import__("fabric_settings", globals(), locals(), [], 0).FABRIC
        try:
            env.settings = global_conf # save settings to switch later
            env.mode = "development"
            load_environment(env.settings[env.mode], False)

        except (KeyError, ValueError):
            raise ImportError
    except (ImportError, AttributeError):
        if not confirm("Warning, no hosts defined: Are you sure you want to continue?"):
            print "\nAborting!"
            exit()


##################
# Template setup #
##################

# Each template gets uploaded at deploy time, only if their
# contents has changed, in which case, the reload command is
# also run.
#
# If a role entry is specified, the template is uploaded only for the given role: application, cron

templates = {
    "nginx": {
        "local_path": "deploy/nginx.conf",
        "remote_path": "/etc/nginx/sites-enabled/%(proj_name)s.conf",
        "reload_command": "service nginx restart",
    },
    "supervisor": {
        "local_path": "deploy/supervisor.conf",
        "remote_path": "/etc/supervisor/conf.d/%(proj_name)s.conf",
        "reload_command": "supervisorctl reload",
    },
    "cron": {
        "local_path": "deploy/crontab",
        "remote_path": "/etc/cron.d/%(proj_name)s",
        "owner": "root",
        "mode": "600",
        "role": "cron",
    },
    "gunicorn": {
        "local_path": "deploy/gunicorn.conf.py",
        "remote_path": "%(proj_path)s/gunicorn.conf.py",
    },
    "settings": {
        "local_path": "deploy/%(mode)s_settings.py", # local task changes this filename
        "remote_path": "%(proj_path)s/local_settings.py",
    },
    "celery": {
        "local_path": "deploy/celeryd.conf",
        "remote_path": "/etc/supervisor/conf.d/%(proj_name)s.celery.conf",
        "required_module": "celery",
        "reload_command": "supervisorctl reload",
    },
}

# Additional templates are created in code.
#
#  ssh
#       ~/.ssh/authorized_keys
#       ~/.ssh/config
#       ~/.ssh/%(DEPLOY_SSH_KEY_PATH.basename())s
#
#  ssh_db
#       ~postgres/.ssh/known_hosts
#       ~postgres/.ssh/config
#       ~postgres/.ssh/authorized_keys
#       ~postgres/.ssh/$(DEPLOY_DB_CLUSTER_SSH_KEY_PATH.basename())s
#
#  postgresql
#      /etc/postgresql/9.2/main/postgresql.conf
#      /etc/postgresql/9.2/main/pg_hba.conf
#
#  db_slave
#      /var/lib/postgresql/9.2/main/recovery.conf
#      /var/lib/postgresql/9.2/archive
#
#  database, db_slave
#      /var/tmp/pg_basebackup_%(proj_name)s.tar.bz2


####################
#  Utility Methods #
####################

CONFIG_FILE_NORMAL = "normal"
CONFIG_FILE_RECORDS = "records"

def modify_config_file(remote_path, settings=None, comment_char='#', setter_char='=', type=CONFIG_FILE_NORMAL,
    use_sudo=False, backup=True):

    """
    Alter the settings of an existing remote config file, trying to uncomment found settings, adding them otherwise.

    The config file is loaded to a local temporary file, processed, and uploaded back to the ``remote_path``.

    By default, backup is made remotely. Set ``backup`` to False to prevent this behavior.

    Two kinds of config files are currently supported.
    Default config file ``type`` is 'normal' where lines look like this:

        # - Connection Settings -
        listen_addresses = 'localhost' # what IP address(es) to listen on;
        #port = 5432

    The ``settings`` must be a list of tuples :

        settings = [('listen_addresses', "'*'"), ('port', '5433')]

    The script will change the config file to :

        # - Connection Settings -
        listen_addresses = '*' # what IP address(es) to listen on;
        port = 5433

    Notice how the line with the port setting was uncommented and set to a new value.
    If an existing line is not found, it is added.

    The ``comment_char`` defaults to '#' and ``setter_char``defaults to '='.

    A basic 'records' config file ``type`` is also supported. Ex: an pg_hba.conf file :

    # TYPE  DATABASE        USER            ADDRESS                 METHOD

    # "local" is for Unix domain socket connections only
    local   all             all                                     peer
    # IPv4 local connections:
    host    all             all             127.0.0.1/32            md5
    # IPv6 local connections:
    host    all             all             ::1/128                 md5
    # Allow replication connections from localhost, by a user with the
    # replication privilege.
    #local   replication     postgres                                peer
    #host    replication     postgres        127.0.0.1/32            md5
    #host    replication     postgres        ::1/128                 md5


    In such a file, we would change the IPv4 local connection to 'trust' and enable local replication like this :
    modify_config_file('/etc/postgresql/9.1/main/pg_hba.conf',
                [('host', 'all', 'all', '127.0.0.1/32','trust'),
                ('local', 'replication', '', 'peer')], type="records")

    By default, the file will be copied to ``remote_path`` as the logged-in
    user; specify ``use_sudo=True`` to use `sudo` instead.

    Downloaded from https://github.com/fabric/fabric/pull/658

    """
    changes_done = set()
    options = dict(comment_char=comment_char, setter_char=setter_char)

    # Create a temporary file
    with tempfile.NamedTemporaryFile(delete=True) as f:

        # Download the remote file into the temporary file
        a = get(remote_path, f)

        # Rewind the file to the beginning
        f.file.seek(0)

        # We're going to read each line and put them into newlines
        newlines = list()

        for line in f.file.readlines():

            # Compare each line with the wanted changes
            for setting in settings:
                # Check for a possible mistake of the user
                if not isinstance(setting, list) and not isinstance(setting, tuple):
                    tb = traceback.format_exc()
                    abort(tb + "\nChanges must be a tuple/list of tuples/lists.\n%s is not a list nor a tuple." % (setting,))

                # Case of a normal config file : each line is of the kind some_variable = some_value
                if type == CONFIG_FILE_NORMAL:
                    key, value = setting

                    # Search for a line of the form : some_variable = some_value
                    # or commented: #some_variable = some_value
                    # We preserve the ending \n.
                    search = ("\s*(?P<is_commented>%(comment_char)s*)(?P<left_chunk>\s*"+_escape_for_regex(key)
                              +"\s*%(setter_char)s\s*)(?P<right_chunk>\s*.*\n*)") % options
                    m = re.match(search, line)

                    if m:
                        # A match was found. Check for a comment at the end of the line,
                        # like some_variable = some_value # this is some comment
                        r = m.groupdict()
                        right_chunk = r['right_chunk']
                        comment = "\n"
                        if right_chunk is not None:
                            # We will capture the comment at the end of the line, if any
                            m2 = re.match("(?P<old_value>.*?)(?P<comment>\s*%(comment_char)s+.*\n*)" % options, right_chunk)
                            if m2:
                                comment = m2.groupdict()['comment']

                        puts('Found line:     ' + line, end='')

                        # It's not rare to find multiple commented versions of a setting in a config file.
                        # If the change was applied in the loop before, make sure we don't set the variable twice in
                        # this case.
                        if setting in changes_done:
                            if r['is_commented']:
                                # This is a commented setting, leave it so.
                                puts('Left it commented.')
                            else:
                                # This is a setting that's not commented, but we set a similar setting earlier in the
                                # loop either because we found a commented version and uncommented it then  applied the
                                # new value, or because it was a duplicate.
                                # In any case, we comment this new setting so our first setting takes precedence.
                                puts('Commented it:   '+red(comment_char) + line)
                                # Recreate the line using the new value set by the user
                                line = comment_char + line
                        else:
                            # Recreate the line using the new value set by the user
                            line = r['left_chunk'] + str(value) + comment

                            puts('Replaced with:  '+r['left_chunk'] + red(value) + comment)

                        # Mark the change as done
                        changes_done.add(setting)

                elif type == CONFIG_FILE_RECORDS:
                    escaped_change = [_escape_for_regex(t) for t in setting]
                    search = ("\s*(?P<is_commented>%(comment_char)s*)(?P<first_values>\s*"
                              + "(\s+)".join(escaped_change[:-1]+[')(?P<right_chunk>.*\n*)'])) % options

                    m = re.match(search, line)
                    if m:
                        r = m.groupdict()
                        right_chunk = r['right_chunk']
                        comment = "\n"
                        if right_chunk is not None:
                            # We will capture the comment at the end of the line, if any
                            m2 = re.match(".*?(?P<comment>\s*%(comment_char)s+.*\n*)" % options, right_chunk)
                            if m2:
                                comment = m2.groupdict()['comment']

                        puts('Found line:     ' + line, end='')

                        if setting in changes_done:
                            if r['is_commented']:
                                puts('Left it commented')
                            else:
                                puts('Commented it:   '+red(comment_char) + line)
                                # Recreate the line using the new value set by the user
                                line = comment_char + line
                        else:
                            # Recreate the line using the new value set by the user
                            line = r['first_values'] + str(setting[-1]) + comment
                            puts('Replaced with:  '+ r['first_values'] + red(str(setting[-1])) + comment)


                        changes_done.add(setting)

            # append the line
            newlines.append(line)

        # So far, some changes have been satisfied : those that were possible by removing the comment on existing lines
        # and setting a new value.
        # Let's now add new lines for remaining changes.
        for setting in settings:
            if not setting in changes_done:
                if type == CONFIG_FILE_NORMAL:
                    line = "%s %s %s\n" % (setting[0], setter_char, setting[1])
                    print line
                    newlines.append(line)
                elif type == CONFIG_FILE_RECORDS:
                    line = "\t".join(setting) + "\n"
                    print line
                    newlines.append(line)

        func = use_sudo and sudo or run

        if backup:
            func("cp %s{,.bak}" % remote_path)

        newlines.append("")

        with tempfile.NamedTemporaryFile(delete=True) as new_file:
            new_file.writelines(newlines)

            # Upload the file.
            put(
                local_path=new_file,
                remote_path=remote_path,
                use_sudo=use_sudo
                #mirror_local_mode=mirror_local_mode,
                #mode=mode
            )


def append_to_remote_file_top(remote_filename, line):
    """
    Replaces the given line with the same at the top of the file.
    """
    if not exists(remote_filename):
        append(remote_filename, line)
    else:
        # read the file and look for the string not using regex
        test = run("cat %s" % remote_filename, show=False)
        if test.find(line) != -1:
            # this code moves the following line to the top of the file
            file = StringIO()
            get(remote_filename, file)
            new_file = [line]
            new_file.extend([l for l in file.getvalue().split("\n") if l != line])
            new_text = "\n".join(new_file) + "\n"
            put(StringIO(new_text), remote_filename)
        else:
            append(remote_filename, line)


def fancy_append(remote_filename, line):
    """
    Fixes append() because it duplicates content of authorized_keys. Returns False if no operation, True if a file was appended.
    """
    if not exists(remote_filename):
        append(remote_filename, line)
    else:
        # read the file and look for the string not using regex
        test = run("cat %s" % remote_filename, show=False)
        if test.find(line) != -1:
            return True
        else:
            append(remote_filename, line)


def check_requirements_file(module):
    """
    Checks a requirements file to see if a module is installed.
    """
    with open(env.reqs_path, "r") as f:
        return f.read().find(module) != -1

######################################
# Context for virtualenv and project #
######################################


@contextmanager
def update_changed_requirements():
    """
    Checks for changes in the requirements file across an update,
    and gets new requirements if changes have occurred.
    """
    reqs_path = join(env.proj_path, env.reqs_path)
    get_reqs = lambda: run("cat %s" % reqs_path, show=False)
    old_reqs = get_reqs() if env.reqs_path else ""
    yield
    if old_reqs:
        new_reqs = get_reqs()
        if old_reqs == new_reqs:
            # Unpinned requirements should always be checked.
            for req in new_reqs.split("\n"):
                if req.startswith("-e"):
                    if "@" not in req:
                        # Editable requirement without pinned commit.
                        break
                elif req.strip() and not req.startswith("#"):
                    if not set(">=<") & set(req):
                        # PyPI requirement without version.
                        break
            else:
                # All requirements are pinned.
                return
        pip("-r %s/%s" % (env.proj_path, env.reqs_path))


###########################################
# Utils and wrappers for various commands #
###########################################

def _print(output):
    print
    print output
    print


@task
@roles('application','cron','database','db_slave')
def run(command, show=True):
    """
    Runs a shell comand on the remote server.
    """
    if show:
        print_command(command)
    with hide("running"):
        return _run(command)


def get_templates():
    """
    Returns each of the templates with env vars injected.
    """
    injected = {}
    for name, data in templates.items():
        injected[name] = dict([(k, v % env) for k, v in data.items()])
    return injected


def upload_template_and_reload(name):
    """
    Uploads a template only if it has changed, and if so, reload a
    related service.
    """
    template = get_templates()[name]
    local_path = template["local_path"]
    remote_path = template["remote_path"]
    reload_command = template.get("reload_command")
    owner = template.get("owner")
    mode = template.get("mode")
    remote_data = ""
    required_module = template.get("required_module")

    # block uploading certain files (e.g., celery configuration) if it is not installed in requirements
    if required_module and not check_requirements_file(required_module):
        return

    if exists(remote_path):
        with hide("stdout"):
            remote_data = sudo("cat %s" % remote_path, show=False)
    print "local_path: %s " % local_path
    with open(local_path, "r") as f:
        local_data = f.read()
        # Escape all non-string-formatting-placeholder occurrences of '%':
        local_data = re.sub(r"%(?!\(\w+\)s)", "%%", local_data)
        if "%(db_pass)s" in local_data:
            env.db_pass = db_pass()
        local_data %= env
    clean = lambda s: s.replace("\n", "").replace("\r", "").strip()
    if clean(remote_data) == clean(local_data):
        return
    upload_template(local_path, remote_path, env, use_sudo=True, backup=False)
    if owner:
        sudo("chown %s %s" % (owner, remote_path))
    if mode:
        sudo("chmod %s %s" % (mode, remote_path))
    if reload_command:
        sudo(reload_command)


def db_pass():
    """
    Prompts for the database password if unknown.
    """
    if not env.db_pass:
        env.db_pass = getpass("Enter the database password: ")
        if not check_db_password(env.db_pass):
            abort("The password you provided contains disallowed special characters")

    return env.db_pass


@task
@roles('application','cron','database','db_slave')
def apt(packages):
    """
    Installs one or more system packages via apt.
    """
    return sudo("apt-get install -y -q " + packages)


def postgres(command):
    """
    Runs the given command as the postgres user.
    """
    show = not command.startswith("psql")
    return run("sudo -u root sudo -u postgres %s" % command, show=show)


@task
@roles('database')
def psql(sql, show=True):
    """
    Runs SQL against the project's database.
    """
    out = postgres('psql -c "%s"' % sql)
{'fraction_non_alphanumeric': 0.11908939285636927, 'fraction_numerical': 0.0054584443433621415, 'mean_word_length': 2.131520043410432, 'pattern_counts': {'":': 54, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 14, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5880765', 'n_tokens_mistral': 13831, 'n_tokens_neox': 13114, 'n_words': 3776}