<commit_before>"""Main Fabric deployment file for BioLinux distributionrepo=.

This installs a standard set of useful biological applications on a remote
server. It is designed for bootstrapping a machine from scratch, as with new
Amazon EC2 instances.

Usage:

    fab -H hostname -i private_key_file install_biolinux

which will call into the 'install_biolinux' method below. See the README for
more examples.

Requires:
    Fabric http://docs.fabfile.org
    PyYAML http://pyyaml.org/wiki/PyYAMLDocumentation
"""
import os
import sys
import subprocess

from fabric.main import load_settings
from fabric.api import *
from fabric.contrib.files import *
import yaml
import logging

# ## General setup
env.config_dir = os.path.join(os.path.dirname(__file__), "config")

# use global cloudbio directory if installed, or utilize local if not
try:
    import cloudbio
except ImportError:
    sys.path.append(os.path.dirname(__file__))

def _setup_logging():
    env.logger = logging.getLogger("cloudbiolinux")
    env.logger.setLevel(logging.DEBUG)

    ch = logging.StreamHandler()
    ch.setLevel(logging.DEBUG)
    # create formatter
    formatter = logging.Formatter('%(name)s %(levelname)s: %(message)s')
    # add formatter to ch
    ch.setFormatter(formatter)
    env.logger.addHandler(ch)

# ### Configuration details for different server types

def _setup_edition():
    """Setup one of the BioLinux editions (which are derived from
       the Edition base class)
    """
    # fetch Edition from environment and load relevant class. Use
    # an existing edition, if possible, and override behaviour through
    # the Flavor mechanism.
    edition = env.get("edition", None)
    if edition is None:
        from cloudbio.edition import Edition
        env.edition = Edition(env)
    elif edition == 'minimal':
        from cloudbio.edition.minimal import Minimal
        env.edition = Minimal(env)
    elif edition == 'bionode':
        from cloudbio.edition.bionode import BioNode
        env.edition = BioNode(env)
    else:
        raise ValueError("Unknown edition: %s" % edition)
    env.logger.debug("%s %s" % (env.edition.name, env.edition.version))
    env.logger.info("This is a %s" % env.edition.short_name)

def _setup_flavor(flavor):
    """Setup flavor
    """
    if flavor != None:
        env.logger.info("Flavor %s" % flavor)
        # to be filled in ...
        env.logger.info("This is a %s" % env.cur_flavor.name)
    else:
        env.logger.info("No flavor defined")

def _setup_distribution_environment():
    """Setup distribution environment
    """
    env.logger.info("Distribution %s" % env.distribution)
    # specialization booleans, simplifying logic (somewhat)
    # these are the only 'special' boolean switches allowed, they can
    # also be moved into the 'edition' logic:
    env.debian = False       # is it pure Debian?
    env.ubuntu = False       # is it pure Ubuntu?
    env.centos = False       # is it pure CentOS?
    env.deb_derived = False  # is it Debian derived?

    if env.hosts == ["vagrant"]:
        _setup_vagrant_environment()
    elif env.hosts == ["localhost"]:
        _setup_local_environment()
    if env.distribution == "ubuntu":
        env.ubuntu = True
        env.deb_derived = True
        _setup_ubuntu()
    elif env.distribution == "centos":
        env.centos = True
        _setup_centos()
    elif env.distribution == "debian":
        env.debian = True
        env.deb_derived = True
        _setup_debian()
    else:
        raise ValueError("Unexpected distribution %s" % env.distribution)
    _expand_shell_paths()

def _validate_target_distribution():
    """Check target matches environment setting (for sanity)

    Throws exception on error
    """
    env.logger.debug("Checking target distribution %s",env.distribution)
    if env.debian:
        tag = run("cat /proc/version")
        if tag.find('ebian') == -1:
           raise ValueError("Debian does not match target, are you using correct fabconfig?")
    elif env.ubuntu:
        tag = run("cat /proc/version")
        if tag.find('buntu') == -1:
           raise ValueError("Ubuntu does not match target, are you using correct fabconfig?")
    else:
        env.logger.debug("Unknown target distro")

def _setup_ubuntu():
    env.logger.info("Ubuntu setup")
    if not env.ubuntu:
       raise ValueError("Target is not Ubuntu")
    shared_sources = _setup_deb_general()
    # package information. This is ubuntu/debian based and could be generalized.
    version = env.dist_name
    sources = [
      "deb http://us.archive.ubuntu.com/ubuntu/ %s universe",
      "deb-src http://us.archive.ubuntu.com/ubuntu/ %s universe",
      "deb http://us.archive.ubuntu.com/ubuntu/ %s-updates universe",
      "deb-src http://us.archive.ubuntu.com/ubuntu/ %s-updates universe",
      "deb http://us.archive.ubuntu.com/ubuntu/ %s multiverse",
      "deb-src http://us.archive.ubuntu.com/ubuntu/ %s multiverse",
      "deb http://us.archive.ubuntu.com/ubuntu/ %s-updates multiverse",
      "deb-src http://us.archive.ubuntu.com/ubuntu/ %s-updates multiverse",
      "deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen", # mongodb
      "deb http://cran.stat.ucla.edu/bin/linux/ubuntu %s/", # lastest R versions
      "deb http://archive.cloudera.com/debian maverick-cdh3 contrib", # Hadoop
      "deb http://archive.canonical.com/ubuntu maverick partner", # sun-java
    ] + shared_sources
    env.std_sources = _add_source_versions(version, sources)

def _setup_debian():
    env.logger.info("Debian setup")
    if not env.debian:
       raise ValueError("Target is not pure Debian")
    shared_sources = _setup_deb_general()
    version = env.dist_name
    if not env.get('debian_repository'):
      main_repository = 'http://ftp.us.debian.org/debian/'
    else:
      main_repository = env.debian_repository

    sources = [
        "deb {repo} %s main contrib non-free".format(repo=main_repository),
        "deb {repo} %s-updates main contrib non-free".format(repo=main_repository)
    ]
    if env.edition.short_name != 'minimal':
        sources = sources + [
          "deb http://downloads-distro.mongodb.org/repo/debian-sysvinit dist 10gen", # mongodb
          "deb http://cran.stat.ucla.edu/bin/linux/debian %s-cran/", # latest R versions
          "deb http://archive.cloudera.com/debian lenny-cdh3 contrib", # Hadoop
        ] + shared_sources
    env.std_sources = _add_source_versions(version, sources)

def _setup_deb_general():
    """Shared settings for different debian based/derived distributions.
    """
    env.logger.debug("Debian-shared setup")
    env.sources_file = "/etc/apt/sources.list"
    env.python_version_ext = ""
    if not env.has_key("java_home"):
        # XXX look for a way to find JAVA_HOME automatically
        env.java_home = "/usr/lib/jvm/java-6-openjdk"
    shared_sources = [
      "deb http://nebc.nox.ac.uk/bio-linux/ unstable bio-linux", # Bio-Linux
    ]
    if env.edition.include_oracle_virtualbox:
        # virtualbox (non-free, otherwise use virtualbox-ose instead)
        shared_sources.append('deb http://download.virtualbox.org/virtualbox/debian %s contrib')
    if env.edition.include_freenx:
        # this arguably belongs in _setup_ubuntu:<commit_msg>Final reduction for Minimal edition - update runs in 8 secs on Thinkpad X300<commit_after>"""Main Fabric deployment file for BioLinux distributionrepo=.

This installs a standard set of useful biological applications on a remote
server. It is designed for bootstrapping a machine from scratch, as with new
Amazon EC2 instances.

Usage:

    fab -H hostname -i private_key_file install_biolinux

which will call into the 'install_biolinux' method below. See the README for
more examples.

Requires:
    Fabric http://docs.fabfile.org
    PyYAML http://pyyaml.org/wiki/PyYAMLDocumentation
"""
import os
import sys
import subprocess

from fabric.main import load_settings
from fabric.api import *
from fabric.contrib.files import *
import yaml
import logging

# ## General setup
env.config_dir = os.path.join(os.path.dirname(__file__), "config")

# use global cloudbio directory if installed, or utilize local if not
try:
    import cloudbio
except ImportError:
    sys.path.append(os.path.dirname(__file__))

def _setup_logging():
    env.logger = logging.getLogger("cloudbiolinux")
    env.logger.setLevel(logging.DEBUG)

    ch = logging.StreamHandler()
    ch.setLevel(logging.DEBUG)
    # create formatter
    formatter = logging.Formatter('%(name)s %(levelname)s: %(message)s')
    # add formatter to ch
    ch.setFormatter(formatter)
    env.logger.addHandler(ch)

# ### Configuration details for different server types

def _setup_edition():
    """Setup one of the BioLinux editions (which are derived from
       the Edition base class)
    """
    # fetch Edition from environment and load relevant class. Use
    # an existing edition, if possible, and override behaviour through
    # the Flavor mechanism.
    edition = env.get("edition", None)
    if edition is None:
        from cloudbio.edition import Edition
        env.edition = Edition(env)
    elif edition == 'minimal':
        from cloudbio.edition.minimal import Minimal
        env.edition = Minimal(env)
    elif edition == 'bionode':
        from cloudbio.edition.bionode import BioNode
        env.edition = BioNode(env)
    else:
        raise ValueError("Unknown edition: %s" % edition)
    env.logger.debug("%s %s" % (env.edition.name, env.edition.version))
    env.logger.info("This is a %s" % env.edition.short_name)

def _setup_flavor(flavor):
    """Setup flavor
    """
    if flavor != None:
        env.logger.info("Flavor %s" % flavor)
        # to be filled in ...
        env.logger.info("This is a %s" % env.cur_flavor.name)
    else:
        env.logger.info("No flavor defined")

def _setup_distribution_environment():
    """Setup distribution environment
    """
    env.logger.info("Distribution %s" % env.distribution)
    # specialization booleans, simplifying logic (somewhat)
    # these are the only 'special' boolean switches allowed, they can
    # also be moved into the 'edition' logic:
    env.debian = False       # is it pure Debian?
    env.ubuntu = False       # is it pure Ubuntu?
    env.centos = False       # is it pure CentOS?
    env.deb_derived = False  # is it Debian derived?

    if env.hosts == ["vagrant"]:
        _setup_vagrant_environment()
    elif env.hosts == ["localhost"]:
        _setup_local_environment()
    if env.distribution == "ubuntu":
        env.ubuntu = True
        env.deb_derived = True
        _setup_ubuntu()
    elif env.distribution == "centos":
        env.centos = True
        _setup_centos()
    elif env.distribution == "debian":
        env.debian = True
        env.deb_derived = True
        _setup_debian()
    else:
        raise ValueError("Unexpected distribution %s" % env.distribution)
    _expand_shell_paths()

def _validate_target_distribution():
    """Check target matches environment setting (for sanity)

    Throws exception on error
    """
    env.logger.debug("Checking target distribution %s",env.distribution)
    if env.debian:
        tag = run("cat /proc/version")
        if tag.find('ebian') == -1:
           raise ValueError("Debian does not match target, are you using correct fabconfig?")
    elif env.ubuntu:
        tag = run("cat /proc/version")
        if tag.find('buntu') == -1:
           raise ValueError("Ubuntu does not match target, are you using correct fabconfig?")
    else:
        env.logger.debug("Unknown target distro")

def _setup_ubuntu():
    env.logger.info("Ubuntu setup")
    if not env.ubuntu:
       raise ValueError("Target is not Ubuntu")
    shared_sources = _setup_deb_general()
    # package information. This is ubuntu/debian based and could be generalized.
    version = env.dist_name
    sources = [
      "deb http://us.archive.ubuntu.com/ubuntu/ %s universe",
      "deb-src http://us.archive.ubuntu.com/ubuntu/ %s universe",
      "deb http://us.archive.ubuntu.com/ubuntu/ %s-updates universe",
      "deb-src http://us.archive.ubuntu.com/ubuntu/ %s-updates universe",
      "deb http://us.archive.ubuntu.com/ubuntu/ %s multiverse",
      "deb-src http://us.archive.ubuntu.com/ubuntu/ %s multiverse",
      "deb http://us.archive.ubuntu.com/ubuntu/ %s-updates multiverse",
      "deb-src http://us.archive.ubuntu.com/ubuntu/ %s-updates multiverse",
      "deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen", # mongodb
      "deb http://cran.stat.ucla.edu/bin/linux/ubuntu %s/", # lastest R versions
      "deb http://archive.cloudera.com/debian maverick-cdh3 contrib", # Hadoop
      "deb http://archive.canonical.com/ubuntu maverick partner", # sun-java
    ] + shared_sources
    env.std_sources = _add_source_versions(version, sources)

def _setup_debian():
    env.logger.info("Debian setup")
    if not env.debian:
       raise ValueError("Target is not pure Debian")
    shared_sources = _setup_deb_general()
    version = env.dist_name
    if not env.get('debian_repository'):
      main_repository = 'http://ftp.us.debian.org/debian/'
    else:
      main_repository = env.debian_repository

    sources = [
        "deb {repo} %s main contrib non-free".format(repo=main_repository),
        "deb {repo} %s-updates main contrib non-free".format(repo=main_repository)
    ]
    if env.edition.short_name != 'minimal':
        sources = sources + [
          "deb http://downloads-distro.mongodb.org/repo/debian-sysvinit dist 10gen", # mongodb
          "deb http://cran.stat.ucla.edu/bin/linux/debian %s-cran/", # latest R versions
          "deb http://archive.cloudera.com/debian lenny-cdh3 contrib", # Hadoop
        ] + shared_sources
    env.std_sources = _add_source_versions(version, sources)

def _setup_deb_general():
    """Shared settings for different debian based/derived distributions.
    """
    env.logger.debug("Debian-shared setup")
    env.sources_file = "/etc/apt/sources.list"
    env.python_version_ext = ""
    if not env.has_key("java_home"):
        # XXX look for a way to find JAVA_HOME automatically
        env.java_home = "/usr/lib/jvm/java-6-openjdk"
    shared_sources = []
    if env.edition.name != 'minimal':
        shared_sources = [
          "deb http://nebc.nox.ac.uk/bio-linux/ unstable bio-linux", # Bio-Linux
        ]
    if env.edition.include_oracle_virtualbox:
        # virtualbox (non-free, otherwise use virtualbox-ose instead)
        shared_sources.append('deb http://download.virtualbox.org/virtualbox/debian %s contrib')
    if env.edition.include_freenx:
        # this arguably belongs in _setup_ubuntu:
{'fraction_non_alphanumeric': 0.11421371654294081, 'fraction_numerical': 0.0016345433494517469, 'mean_word_length': 3.0296377607025247, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5640522', 'n_tokens_mistral': 4722, 'n_tokens_neox': 4371, 'n_words': 1185}