<commit_before>#!/usr/bin/env python
"""Automatically install required tools and data to run bcbio-nextgen pipelines.

This automates the steps required for installation and setup to make it
easier to get started with bcbio-nextgen. The defaults provide data files
for human variant calling.

Requires: PyYAML, fabric
"""
import argparse
import contextlib
import datetime
import os
import urllib2
import shutil
import subprocess
import sys

import yaml

remotes = {"system_config":
           "https://raw.github.com/chapmanb/bcbio-nextgen/master/config/bcbio_system.yaml",
           "requirements":
           "https://raw.github.com/chapmanb/bcbio-nextgen/master/requirements.txt",
           "virtualenv":
           "https://raw.github.com/pypa/virtualenv/master/virtualenv.py"}

def main(args):
    check_dependencies()
    work_dir = os.path.join(os.getcwd(), "tmpbcbio-install")
    if not os.path.exists(work_dir):
        os.makedirs(work_dir)
    os.chdir(work_dir)
    cbl = get_cloudbiolinux()
    fabricrc = write_fabricrc(cbl["fabricrc"], args.tooldir, args.datadir,
                              args.distribution, args.sudo)
    biodata = write_biodata(cbl["biodata"], args.genomes, args.aligners)
    if args.install_tools:
        print "Installing tools..."
        install_tools(cbl["tool_fabfile"], fabricrc)
    print "Installing data..."
    if args.install_data:
        install_data(cbl["data_fabfile"], fabricrc, biodata)
    print "Installing bcbio-nextgen..."
    install_bcbio_nextgen(remotes["requirements"], args.datadir, args.tooldir,
                          args.sudo)
    system_config = write_system_config(remotes["system_config"], args.datadir,
                                        args.tooldir)
    print "Finished: bcbio-nextgen, tools and data installed"
    print " Ready to use system configuration at:\n  %s" % system_config
    print " Tools installed in:\n  %s" % args.tooldir
    print " Genome data installed in:\n  %s" % args.datadir
    shutil.rmtree(work_dir)

def install_bcbio_nextgen(requirements, datadir, tooldir, use_sudo):
    """Install a virtualenv containing bcbio_nextgen depdencies.
    """
    virtualenv_dir = os.path.join(datadir, "bcbio-nextgen-virtualenv")
    if not os.path.exists(virtualenv_dir):
        subprocess.check_call(["wget", remotes["virtualenv"]])
        subprocess.check_call(["python", "virtualenv.py", "--no-site-packages",
                               "--distribute", virtualenv_dir])
        os.remove("virtualenv.py")
    pip_cmd = os.path.join(virtualenv_dir, "bin", "pip")
    subprocess.check_call([pip_cmd, "install", "--upgrade", "distribute"])
    subprocess.check_call([pip_cmd, "install", "-r", requirements])
    for script in ["bcbio_nextgen.py", "bam_to_wiggle.py"]:
        final_script = os.path.join(tooldir, "bin", script)
        ve_script = os.path.join(virtualenv_dir, "bin", script)
        if not os.path.exists(final_script):
            sudo_cmd = ["sudo"] if use_sudo else []
            subprocess.check_call(sudo_cmd + ["mkdir", "-p", os.path.dirname(final_script)])
            cmd = ["ln", "-s", ve_script, final_script]
            subprocess.check_call(sudo_cmd + cmd)

def install_tools(fabfile, fabricrc):
    subprocess.check_call(["fab", "-f", fabfile, "-H", "localhost",
                           "-c", fabricrc,
                           "install_biolinux:flavor=ngs_pipeline"])

def install_data(fabfile, fabricrc, biodata):<commit_msg>Require cython installation before bcbio-nextgen requirements for pybedtools dependency<commit_after>#!/usr/bin/env python
"""Automatically install required tools and data to run bcbio-nextgen pipelines.

This automates the steps required for installation and setup to make it
easier to get started with bcbio-nextgen. The defaults provide data files
for human variant calling.

Requires: PyYAML, fabric
"""
import argparse
import contextlib
import datetime
import os
import urllib2
import shutil
import subprocess
import sys

import yaml

remotes = {"system_config":
           "https://raw.github.com/chapmanb/bcbio-nextgen/master/config/bcbio_system.yaml",
           "requirements":
           "https://raw.github.com/chapmanb/bcbio-nextgen/master/requirements.txt",
           "virtualenv":
           "https://raw.github.com/pypa/virtualenv/master/virtualenv.py"}

def main(args):
    check_dependencies()
    work_dir = os.path.join(os.getcwd(), "tmpbcbio-install")
    if not os.path.exists(work_dir):
        os.makedirs(work_dir)
    os.chdir(work_dir)
    cbl = get_cloudbiolinux()
    fabricrc = write_fabricrc(cbl["fabricrc"], args.tooldir, args.datadir,
                              args.distribution, args.sudo)
    biodata = write_biodata(cbl["biodata"], args.genomes, args.aligners)
    if args.install_tools:
        print "Installing tools..."
        install_tools(cbl["tool_fabfile"], fabricrc)
    print "Installing data..."
    if args.install_data:
        install_data(cbl["data_fabfile"], fabricrc, biodata)
    print "Installing bcbio-nextgen..."
    install_bcbio_nextgen(remotes["requirements"], args.datadir, args.tooldir,
                          args.sudo)
    system_config = write_system_config(remotes["system_config"], args.datadir,
                                        args.tooldir)
    print "Finished: bcbio-nextgen, tools and data installed"
    print " Ready to use system configuration at:\n  %s" % system_config
    print " Tools installed in:\n  %s" % args.tooldir
    print " Genome data installed in:\n  %s" % args.datadir
    shutil.rmtree(work_dir)

def install_bcbio_nextgen(requirements, datadir, tooldir, use_sudo):
    """Install a virtualenv containing bcbio_nextgen depdencies.
    """
    virtualenv_dir = os.path.join(datadir, "bcbio-nextgen-virtualenv")
    if not os.path.exists(virtualenv_dir):
        subprocess.check_call(["wget", remotes["virtualenv"]])
        subprocess.check_call(["python", "virtualenv.py", "--no-site-packages",
                               "--distribute", virtualenv_dir])
        os.remove("virtualenv.py")
    pip_cmd = os.path.join(virtualenv_dir, "bin", "pip")
    subprocess.check_call([pip_cmd, "install", "--upgrade", "distribute"])
    subprocess.check_call([pip_cmd, "install", "--upgrade", "cython"])
    subprocess.check_call([pip_cmd, "install", "-r", requirements])
    for script in ["bcbio_nextgen.py", "bam_to_wiggle.py"]:
        final_script = os.path.join(tooldir, "bin", script)
        ve_script = os.path.join(virtualenv_dir, "bin", script)
        if not os.path.exists(final_script):
            sudo_cmd = ["sudo"] if use_sudo else []
            subprocess.check_call(sudo_cmd + ["mkdir", "-p", os.path.dirname(final_script)])
            cmd = ["ln", "-s", ve_script, final_script]
            subprocess.check_call(sudo_cmd + cmd)

def install_tools(fabfile, fabricrc):
    subprocess.check_call(["fab", "-f", fabfile, "-H", "localhost",
                           "-c", fabricrc,
                           "install_biolinux:flavor=ngs_pipeline"])

def install_data(fabfile, fabricrc, biodata):
{'fraction_non_alphanumeric': 0.12572315507266826, 'fraction_numerical': 0.00028220685762664035, 'mean_word_length': 3.0994794679005206, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2741476', 'n_tokens_mistral': 2290, 'n_tokens_neox': 2237, 'n_words': 471}