<commit_before>"""Install packages via the MacOSX Homebrew and Linux Linuxbrew package manager.
https://github.com/mxcl/homebrew
https://github.com/Homebrew/linuxbrew
"""
import contextlib
from distutils.version import LooseVersion
import os
import sys

from cloudbio.custom import system, shared
from cloudbio.flavor.config import get_config_file
from cloudbio.fabutils import quiet, find_cmd
from cloudbio.package import cpan
from cloudbio.package.shared import _yaml_to_packages

from fabric.api import cd, settings

BOTTLE_URL = "https://s3.amazonaws.com/cloudbiolinux/brew_bottles/{pkg}-{version}.x86_64-linux.bottle.tar.gz"
BOTTLE_SUPPORTED = set(["isaac-aligner", "isaac-variant-caller", "cmake"])

def install_packages(env, to_install=None, packages=None):
    """Install packages using the home brew package manager.

    Handles upgrading brew, tapping required repositories and installing or upgrading
    packages as appropriate.

    `to_install` is a CloudBioLinux compatible set of top level items to add,
    alternatively `packages` is a list of raw package names.
    """
    config_file = get_config_file(env, "packages-homebrew.yaml")
    if to_install:
        (packages, _) = _yaml_to_packages(config_file.base, to_install, config_file.dist)
    # if we have no packages to install, do not try to install or update brew
    if len(packages) == 0:
        return
    system.install_homebrew(env)
    brew_cmd = _brew_cmd(env)
    formula_repos = ["homebrew/science", "chapmanb/cbl"]
    current_taps = set([x.strip() for x in env.safe_run_output("%s tap" % brew_cmd).split()])
    _safe_update(env, brew_cmd, formula_repos, current_taps)
    for repo in formula_repos:
        if repo not in current_taps:
            env.safe_run("%s tap %s" % (brew_cmd, repo))
    env.safe_run("%s tap --repair" % brew_cmd)
    ipkgs = {"outdated": set([x.strip() for x in env.safe_run_output("%s outdated" % brew_cmd).split()]),
             "current": _get_current_pkgs(env, brew_cmd)}
    _install_brew_baseline(env, brew_cmd, ipkgs, packages)
    ipkgs = {"outdated": set([x.strip() for x in env.safe_run_output("%s outdated" % brew_cmd).split()]),
             "current": _get_current_pkgs(env, brew_cmd)}
    for pkg_str in packages:
        _install_pkg(env, pkg_str, brew_cmd, ipkgs)
    for pkg_str in ["pkg-config", "openssl", "cmake"]:
        _safe_unlink_pkg(env, pkg_str, brew_cmd)
    for pkg_str in ["curl"]:
        _safe_uninstall_pkg(env, pkg_str, brew_cmd)

def _safe_update(env, brew_cmd, formula_repos, cur_taps):
    """Revert any taps if we fail to update due to local changes.
    """
    with _git_stash(env, brew_cmd):
        with quiet():
            with settings(warn_only=True):
                out = env.safe_run("%s update" % brew_cmd)
        if out.failed:
            for repo in formula_repos:
                if repo in cur_taps:
                    env.safe_run("%s untap %s" % (brew_cmd, repo))
            env.safe_run("%s update" % brew_cmd)

@contextlib.contextmanager
def _git_stash(env, brew_cmd):
    """Perform a safe git stash around an update.

    This circumvents brews internal stash approach which doesn't work on older versions
    of git and is sensitive to missing config.emails.
    """
    brew_prefix = env.safe_run_output("{brew_cmd} --prefix".format(**locals()))
    with cd(brew_prefix):
        with settings(warn_only=True):
            check_diff = env.safe_run("git diff --quiet")
    if check_diff.return_code > 0:
        with cd(brew_prefix):
            env.safe_run("git config user.email '<EMAIL>'")
            env.safe_run("git stash --quiet")
    try:
        yield None
    finally:
        if check_diff.return_code > 0:
            with cd(brew_prefix):
                env.safe_run("git stash pop --quiet")

def _get_current_pkgs(env, brew_cmd):
    out = {}
    with quiet():
        which_out = env.safe_run_output("{brew_cmd} list --versions".format(**locals()))
    for line in which_out.split("\n"):
        if line:
            parts = line.rstrip().split()
            if len(parts) == 2:
                pkg, version = line.rstrip().split()
                if pkg.endswith(":"):
                    pkg = pkg[:-1]
                out[pkg] = version
    return out

def _safe_unlink_pkg(env, pkg_str, brew_cmd):
    """Unlink packages which can cause issues with a Linux system.
    """
    with settings(warn_only=True):
        with quiet():<commit_msg>Cleanly skip brew update if git version is too old. chapmanb/bcbio-nextgen#435<commit_after>"""Install packages via the MacOSX Homebrew and Linux Linuxbrew package manager.
https://github.com/mxcl/homebrew
https://github.com/Homebrew/linuxbrew
"""
import contextlib
from distutils.version import LooseVersion
import os
import sys

from cloudbio.custom import system, shared
from cloudbio.flavor.config import get_config_file
from cloudbio.fabutils import quiet, find_cmd
from cloudbio.package import cpan
from cloudbio.package.shared import _yaml_to_packages

from fabric.api import cd, settings

BOTTLE_URL = "https://s3.amazonaws.com/cloudbiolinux/brew_bottles/{pkg}-{version}.x86_64-linux.bottle.tar.gz"
BOTTLE_SUPPORTED = set(["isaac-aligner", "isaac-variant-caller", "cmake"])

def install_packages(env, to_install=None, packages=None):
    """Install packages using the home brew package manager.

    Handles upgrading brew, tapping required repositories and installing or upgrading
    packages as appropriate.

    `to_install` is a CloudBioLinux compatible set of top level items to add,
    alternatively `packages` is a list of raw package names.
    """
    config_file = get_config_file(env, "packages-homebrew.yaml")
    if to_install:
        (packages, _) = _yaml_to_packages(config_file.base, to_install, config_file.dist)
    # if we have no packages to install, do not try to install or update brew
    if len(packages) == 0:
        return
    system.install_homebrew(env)
    brew_cmd = _brew_cmd(env)
    formula_repos = ["homebrew/science", "chapmanb/cbl"]
    current_taps = set([x.strip() for x in env.safe_run_output("%s tap" % brew_cmd).split()])
    _safe_update(env, brew_cmd, formula_repos, current_taps)
    current_taps = set([x.strip() for x in env.safe_run_output("%s tap" % brew_cmd).split()])
    for repo in formula_repos:
        if repo not in current_taps:
            env.safe_run("%s tap %s" % (brew_cmd, repo))
    env.safe_run("%s tap --repair" % brew_cmd)
    ipkgs = {"outdated": set([x.strip() for x in env.safe_run_output("%s outdated" % brew_cmd).split()]),
             "current": _get_current_pkgs(env, brew_cmd)}
    _install_brew_baseline(env, brew_cmd, ipkgs, packages)
    ipkgs = {"outdated": set([x.strip() for x in env.safe_run_output("%s outdated" % brew_cmd).split()]),
             "current": _get_current_pkgs(env, brew_cmd)}
    for pkg_str in packages:
        _install_pkg(env, pkg_str, brew_cmd, ipkgs)
    for pkg_str in ["pkg-config", "openssl", "cmake"]:
        _safe_unlink_pkg(env, pkg_str, brew_cmd)
    for pkg_str in ["curl"]:
        _safe_uninstall_pkg(env, pkg_str, brew_cmd)

def _safe_update(env, brew_cmd, formula_repos, cur_taps):
    """Revert any taps if we fail to update due to local changes.
    """
    with _git_stash(env, brew_cmd):
        with quiet():
            with settings(warn_only=True):
                out = env.safe_run("%s update" % brew_cmd)
        if out.failed:
            for repo in formula_repos:
                if repo in cur_taps:
                    env.safe_run("%s untap %s" % (brew_cmd, repo))
            with settings(warn_only=True):
                out = env.safe_run("%s update" % brew_cmd)
            if out.failed:
                print("\n\nHomebrew update failed.")
                print("You might need to upgrade git by installing inside bcbio with:")
                print("'brew install git --env=inherit --ignore-dependences'\n\n")

@contextlib.contextmanager
def _git_stash(env, brew_cmd):
    """Perform a safe git stash around an update.

    This circumvents brews internal stash approach which doesn't work on older versions
    of git and is sensitive to missing config.emails.
    """
    brew_prefix = env.safe_run_output("{brew_cmd} --prefix".format(**locals()))
    with cd(brew_prefix):
        with quiet():
            with settings(warn_only=True):
                check_diff = env.safe_run("git diff --quiet")
    if check_diff.return_code > 0:
        with cd(brew_prefix):
            env.safe_run("git config user.email '<EMAIL>'")
            env.safe_run("git stash --quiet")
    try:
        yield None
    finally:
        if check_diff.return_code > 0:
            with cd(brew_prefix):
                env.safe_run("git stash pop --quiet")

def _get_current_pkgs(env, brew_cmd):
    out = {}
    with quiet():
        which_out = env.safe_run_output("{brew_cmd} list --versions".format(**locals()))
    for line in which_out.split("\n"):
        if line:
            parts = line.rstrip().split()
            if len(parts) == 2:
                pkg, version = line.rstrip().split()
                if pkg.endswith(":"):
                    pkg = pkg[:-1]
                out[pkg] = version
    return out

def _safe_unlink_pkg(env, pkg_str, brew_cmd):
    """Unlink packages which can cause issues with a Linux system.
    """
    with settings(warn_only=True):
        with quiet():
{'fraction_non_alphanumeric': 0.11372590778404856, 'fraction_numerical': 0.0024491534447875626, 'mean_word_length': 2.9813480288257734, 'pattern_counts': {'":': 10, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3066289', 'n_tokens_mistral': 3043, 'n_tokens_neox': 2921, 'n_words': 807}