<commit_before>#! /usr/bin/env perl

# This script can be run from any directory

use strict;
use Carp;
use Cwd;
use Data::Dumper;
use File::Basename;
use Getopt::Long;

my $usage =<<"End_of_Usage";

Usage: sudo add-comp.pl [ options ] components

Options:
      -d dest_dir  - absolute destination directory (D = /repo/third_party)
      -f           - force reinstall even if component exists
      -t tmp_dir   - temporary directory (D = /mnt/tmp)
      --dry        - dry run: check if modules exist

Compute server components:
      basic        - basic dependencies (apt-get, pip, cpan, etc)
      regular      - regular components (modules to be deployed on all compute nodes)
      special      - special components (large modules: pacbio, allpaths-lg, etc)
      all          - all components

      a5           - A5 pipeline (v.20140604)
      a6           - Modified A5 pipeline (git)
      ale          - ALE reference-free assembler evaluator (git)
      bowtie2      - Bowtie aligner (v2.1)
      bwa          - BWA aligner (git)
      discovar     - Discovar assembler (FTP latest)
      fastqc       - FastQC quality control (v0.10.1)
      fastx        - FastX preprocessing toolkit (v0.0.13)
      gam_ngs      - GAM-NGS assembler merger (git)
      idba         - IDBA_UD assembler (v1.1.1)
      jgi_rqc      - JGI rolling QC (git)
      kiki         - Kiki assembler (git)
      kmergenie    - KmerGenie (v1.6663)
      masurca      - MaSuRCA assembler (v2.2.1)
      pacbio       - SMRT Analysis Software (v2.1.1)
      prodigal     - Prodigal Prokaryotic Gene Prediction (v2.60)
      quast        - QUAST assembly evaluator (v2.3)
      ray          - Ray assembler (git)
      reapr        - REAPR reference-free evaluator (v1.0.17)
      seqtk        - Modified Seqtk preprocessing toolkit (git)
      solexa       - SolexaQA preprocessing tool (v2.1)
      spades       - SPAdes assembler (v3.1.0)
      velvet       - Velvet assembler (git)

Examples:
      sudo add-comp.pl basic velvet spades
      sudo add-comp.pl -t /space/tmp -d /usr/bin regular

End_of_Usage

my ($help, $dest_dir, $dry_run, $tmp_dir, $force_reinstall);

GetOptions( 'd|dest=s' => \$dest_dir,
            'dry'      => \$dry_run,
            'f|force'  => \$force_reinstall,
            't|tmp=s'  => \$tmp_dir,
            'h|help'   => \$help);

if ($help || @ARGV == 0) { print $usage; exit 0 }

my @regular_comps = qw (basic a5 a6 ale bowtie2 bwa fastqc fastx gam_ngs idba kiki kmergenie masurca quast prodigal ray reapr seqtk solexa spades velvet);
my @special_comps = qw (discovar pacbio jgi_rqc);
my @extra_depends = qw (cmake3);

my @all_comps = (@regular_comps, @special_comps);
my %supported = map { $_ => 1 } (@all_comps, @extra_depends);

my @comps;
for (@ARGV) {
    if (/\ball\b/) {
        @comps = @all_comps; last;
    } elsif (/\bregular\b/) {
        @comps = (@comps, @regular_comps);
    } elsif (/\bspecial\b/) {
        @comps = (@comps, @special_comps);
    } else {
        push @comps, $_;
    }
}


my $curr_dir = cwd();
my $base_dir = dirname(Cwd::abs_path($0));
$dest_dir  ||= "$base_dir/../third_party"; run("mkdir -p $dest_dir") if !$dry_run;
$tmp_dir     = make_tmp_dir($tmp_dir) if !$dry_run;

for my $c (@comps) {
    if (!$supported{$c}) { print "Warning: $c not supported.\n"; next; }
    my $found = !$force_reinstall && check_if_installed($c);
    if ($found) { print "Found component $c, skipping...\n"; next; }

    print "Installing $c...\n";
    next if $dry_run;

    my $func = "install_$c";

    chdir($tmp_dir);
    { no strict 'refs'; &$func(); }
    chdir($curr_dir);
    print "\n";
}


# Core dependencies

# TODO: python: fallback: strip path prefix and check exe in path

sub install_template {
    # we are in the $tmp_dir directory
    # 1. download source files
    # 2. compile
    # 3. copy executables to $dest_dir
}

sub install_basic {
    my @apt = qw(python-nova build-essential python-pip rabbitmq-server git mongodb cmake zlib1g-dev mpich2 samtools openjdk-7-jre subversion python-matplotlib unzip r-base unp cpanminus picard-tools gcc-4.7 g++-4.7 graphviz csh pkg-config sparsehash libboost-all-dev gawk);
    my @pip = qw(pika python-daemon pymongo requests yapsy numpy biopython);

    run("apt-get -y purge openmpi*");
    # run("apt-get -q -y update");
    run("apt-get -y install " . join(" ", @apt));
    run("pip install "        . join(" ", @pip));

    run('curl -L http://cpanmin.us | perl - App::cpanminus');
    run('cpanm File::Spec::Link');

    install_cmake3();
}

sub install_cmake3 {
    # spades requires v2.8.8+
    my ($ver) = `cmake --version` =~ /version (\d[0-9.]+)/;
    return if $ver >= 3.0;
    my $dir = 'cmake-3.0.0';
    my $file = "$dir.tar.gz";
    download($dir, $file, 'http://www.cmake.org/files/v3.0');
    chdir($dir);
    run('./bootstrap; make -j && make install');
}

sub install_a5 {
    run("mkdir -p a5");
    chdir("a5");
    my $dir = "a5_miseq_linux_20140604";
    my $file = "$dir.tar.gz";
    download($dir, $file, "http://sourceforge.net/projects/ngopt/files");
    run("cp -r -T $dir $dest_dir/a5");
}

sub install_a6 {
    git("git://github.com/levinas/a5.git", "a6");
    run("cp -r a6 $dest_dir/");
}

sub install_ale {
    my $dir = 'ale';
    git("git://github.com/sc932/ALE.git");
    run("mkdir -p $dest_dir/$dir");
    run("cd ALE/src; make ALE; cp -r ALE samtools-0.1.19 *.sh *.py *.pl $dest_dir/$dir/");
}

sub install_bowtie2 {
    my $file = "bowtie2-2.1.0-linux-x86_64.zip";
    my $dir = "bowtie2-2.1.0";
    download($dir, $file, "http://sourceforge.net/projects/bowtie-bio/files/bowtie2/2.1.0");
    run("cp -r -T $dir $dest_dir/bowtie2");
}

sub install_bwa {
    git("git://github.com/lh3/bwa.git");
    run("cd bwa; make -j; cp bwa $dest_dir/");
}

sub install_discovar {
    check_gcc();
    my $file = "LATEST_VERSION.tar.gz";
    my $dir = "discovar";
    download($dir, $file, 'ftp://ftp.broadinstitute.org/pub/crd/Discovar/latest_source_code');
    run("mv discovar-* $dir");
    run("cd $dir; ./configure; make -j; cp src/Discovar $dest_dir/discovar");
}

sub install_fastqc {
    my $dir = 'FastQC';
    my $file = 'fastqc_v0.10.1.zip';
    download($dir, $file, 'http://www.bioinformatics.babraham.ac.uk/projects/fastqc');
    run("chmod a+x $dir/fastqc");
    run("cp -r $dir $dest_dir/");
}

sub install_fastx {
    my $dir = 'fastx_toolkit';
    # there is no fastx plugin; so we need this extra check
    if (-e "$dest_dir/$dir/fastx_trimmer") {
        print "Found component fastx, skipping...\n";
        return;
    }
    run("mkdir -p $dir");
    chdir($dir);
    download('bin', 'fastx_toolkit_0.0.13_binaries_Linux_2.6_amd64.tar.bz2', 'http://hannonlab.cshl.edu/fastx_toolkit');
    run("cp -r -T bin $dest_dir/$dir");
}

sub install_gam_ngs {
    my $dir = 'gam-ngs';
    git('git://github.com/vice87/gam-ngs.git');
    run("cd $dir; mkdir -p build; cd build; cmake ..; make -j");
    run("cp -r $dir $dest_dir/");
}

sub install_idba {
    my $dir = 'idba-1.1.1';
    my $file = "$dir.tar.gz";
    download($dir, $file, 'http://hku-idba.googlecode.com/files');
    run("cd $dir; ./configure; make -j");
    run("cp -r -T $dir/bin $dest_dir/idba");
}

sub install_kiki {
    git('git://github.com/GeneAssembly/kiki.git');
    chdir("kiki");
    run("mkdir -p bin; cd bin; cmake ..; make -j ki");
    run("cp bin/ki $dest_dir/");
}

sub install_kmergenie {
    my $dir = 'kmergenie-1.6663';<commit_msg>move openmpi-removal code to install_kiki<commit_after>#! /usr/bin/env perl

# This script can be run from any directory

use strict;
use Carp;
use Cwd;
use Data::Dumper;
use File::Basename;
use Getopt::Long;

my $usage =<<"End_of_Usage";

Usage: sudo add-comp.pl [ options ] components

Options:
      -d dest_dir  - absolute destination directory (D = /repo/third_party)
      -f           - force reinstall even if component exists
      -t tmp_dir   - temporary directory (D = /mnt/tmp)
      --dry        - dry run: check if modules exist

Compute server components:
      basic        - basic dependencies (apt-get, pip, cpan, etc)
      regular      - regular components (modules to be deployed on all compute nodes)
      special      - special components (large modules: pacbio, allpaths-lg, etc)
      all          - all components

      a5           - A5 pipeline (v.20140604)
      a6           - Modified A5 pipeline (git)
      ale          - ALE reference-free assembler evaluator (git)
      bowtie2      - Bowtie aligner (v2.1)
      bwa          - BWA aligner (git)
      discovar     - Discovar assembler (FTP latest)
      fastqc       - FastQC quality control (v0.10.1)
      fastx        - FastX preprocessing toolkit (v0.0.13)
      gam_ngs      - GAM-NGS assembler merger (git)
      idba         - IDBA_UD assembler (v1.1.1)
      jgi_rqc      - JGI rolling QC (git)
      kiki         - Kiki assembler (git)
      kmergenie    - KmerGenie (v1.6663)
      masurca      - MaSuRCA assembler (v2.2.1)
      pacbio       - SMRT Analysis Software (v2.1.1)
      prodigal     - Prodigal Prokaryotic Gene Prediction (v2.60)
      quast        - QUAST assembly evaluator (v2.3)
      ray          - Ray assembler (git)
      reapr        - REAPR reference-free evaluator (v1.0.17)
      seqtk        - Modified Seqtk preprocessing toolkit (git)
      solexa       - SolexaQA preprocessing tool (v2.1)
      spades       - SPAdes assembler (v3.1.0)
      velvet       - Velvet assembler (git)

Examples:
      sudo add-comp.pl basic velvet spades
      sudo add-comp.pl -t /space/tmp -d /usr/bin regular

End_of_Usage

my ($help, $dest_dir, $dry_run, $tmp_dir, $force_reinstall);

GetOptions( 'd|dest=s' => \$dest_dir,
            'dry'      => \$dry_run,
            'f|force'  => \$force_reinstall,
            't|tmp=s'  => \$tmp_dir,
            'h|help'   => \$help);

if ($help || @ARGV == 0) { print $usage; exit 0 }

my @regular_comps = qw (basic a5 a6 ale bowtie2 bwa fastqc fastx gam_ngs idba kiki kmergenie masurca quast prodigal ray reapr seqtk solexa spades velvet);
my @special_comps = qw (discovar pacbio jgi_rqc);
my @extra_depends = qw (cmake3);

my @all_comps = (@regular_comps, @special_comps);
my %supported = map { $_ => 1 } (@all_comps, @extra_depends);

my @comps;
for (@ARGV) {
    if (/\ball\b/) {
        @comps = @all_comps; last;
    } elsif (/\bregular\b/) {
        @comps = (@comps, @regular_comps);
    } elsif (/\bspecial\b/) {
        @comps = (@comps, @special_comps);
    } else {
        push @comps, $_;
    }
}


my $curr_dir = cwd();
my $base_dir = dirname(Cwd::abs_path($0));
$dest_dir  ||= "$base_dir/../third_party"; run("mkdir -p $dest_dir") if !$dry_run;
$tmp_dir     = make_tmp_dir($tmp_dir) if !$dry_run;

for my $c (@comps) {
    if (!$supported{$c}) { print "Warning: $c not supported.\n"; next; }
    my $found = !$force_reinstall && check_if_installed($c);
    if ($found) { print "Found component $c, skipping...\n"; next; }

    print "Installing $c...\n";
    next if $dry_run;

    my $func = "install_$c";

    chdir($tmp_dir);
    { no strict 'refs'; &$func(); }
    chdir($curr_dir);
    print "\n";
}


# Core dependencies

# TODO: python: fallback: strip path prefix and check exe in path

sub install_template {
    # we are in the $tmp_dir directory
    # 1. download source files
    # 2. compile
    # 3. copy executables to $dest_dir
}

sub install_basic {
    my @apt = qw(python-nova build-essential python-pip rabbitmq-server git mongodb cmake zlib1g-dev mpich2 samtools openjdk-7-jre subversion python-matplotlib unzip r-base unp cpanminus picard-tools gcc-4.7 g++-4.7 graphviz csh pkg-config sparsehash libboost-all-dev gawk);
    my @pip = qw(pika python-daemon pymongo requests yapsy numpy biopython);

    # run("apt-get -q -y update");
    run("apt-get -y install " . join(" ", @apt));
    run("pip install "        . join(" ", @pip));

    run('curl -L http://cpanmin.us | perl - App::cpanminus');
    run('cpanm File::Spec::Link');

    install_cmake3();
}

sub install_cmake3 {
    # spades requires v2.8.8+
    my ($ver) = `cmake --version` =~ /version (\d[0-9.]+)/;
    return if $ver >= 3.0;
    my $dir = 'cmake-3.0.0';
    my $file = "$dir.tar.gz";
    download($dir, $file, 'http://www.cmake.org/files/v3.0');
    chdir($dir);
    run('./bootstrap; make -j && make install');
}

sub install_a5 {
    run("mkdir -p a5");
    chdir("a5");
    my $dir = "a5_miseq_linux_20140604";
    my $file = "$dir.tar.gz";
    download($dir, $file, "http://sourceforge.net/projects/ngopt/files");
    run("cp -r -T $dir $dest_dir/a5");
}

sub install_a6 {
    git("git://github.com/levinas/a5.git", "a6");
    run("cp -r a6 $dest_dir/");
}

sub install_ale {
    my $dir = 'ale';
    git("git://github.com/sc932/ALE.git");
    run("mkdir -p $dest_dir/$dir");
    run("cd ALE/src; make ALE; cp -r ALE samtools-0.1.19 *.sh *.py *.pl $dest_dir/$dir/");
}

sub install_bowtie2 {
    my $file = "bowtie2-2.1.0-linux-x86_64.zip";
    my $dir = "bowtie2-2.1.0";
    download($dir, $file, "http://sourceforge.net/projects/bowtie-bio/files/bowtie2/2.1.0");
    run("cp -r -T $dir $dest_dir/bowtie2");
}

sub install_bwa {
    git("git://github.com/lh3/bwa.git");
    run("cd bwa; make -j; cp bwa $dest_dir/");
}

sub install_discovar {
    check_gcc();
    my $file = "LATEST_VERSION.tar.gz";
    my $dir = "discovar";
    download($dir, $file, 'ftp://ftp.broadinstitute.org/pub/crd/Discovar/latest_source_code');
    run("mv discovar-* $dir");
    run("cd $dir; ./configure; make -j; cp src/Discovar $dest_dir/discovar");
}

sub install_fastqc {
    my $dir = 'FastQC';
    my $file = 'fastqc_v0.10.1.zip';
    download($dir, $file, 'http://www.bioinformatics.babraham.ac.uk/projects/fastqc');
    run("chmod a+x $dir/fastqc");
    run("cp -r $dir $dest_dir/");
}

sub install_fastx {
    my $dir = 'fastx_toolkit';
    # there is no fastx plugin; so we need this extra check
    if (-e "$dest_dir/$dir/fastx_trimmer") {
        print "Found component fastx, skipping...\n";
        return;
    }
    run("mkdir -p $dir");
    chdir($dir);
    download('bin', 'fastx_toolkit_0.0.13_binaries_Linux_2.6_amd64.tar.bz2', 'http://hannonlab.cshl.edu/fastx_toolkit');
    run("cp -r -T bin $dest_dir/$dir");
}

sub install_gam_ngs {
    my $dir = 'gam-ngs';
    git('git://github.com/vice87/gam-ngs.git');
    run("cd $dir; mkdir -p build; cd build; cmake ..; make -j");
    run("cp -r $dir $dest_dir/");
}

sub install_idba {
    my $dir = 'idba-1.1.1';
    my $file = "$dir.tar.gz";
    download($dir, $file, 'http://hku-idba.googlecode.com/files');
    run("cd $dir; ./configure; make -j");
    run("cp -r -T $dir/bin $dest_dir/idba");
}

sub install_kiki {
    run("apt-get -y purge openmpi*");
    git('git://github.com/GeneAssembly/kiki.git');
    chdir("kiki");
    run("mkdir -p bin; cd bin; cmake ..; make -j ki");
    run("cp bin/ki $dest_dir/");
}

sub install_kmergenie {
    my $dir = 'kmergenie-1.6663';
{'fraction_non_alphanumeric': 0.1598420982202596, 'fraction_numerical': 0.01993844506891476, 'mean_word_length': 2.5571156592099, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4778594', 'n_tokens_mistral': 5911, 'n_tokens_neox': 5647, 'n_words': 1407}