<commit_before>/**
 * Licensed to Big Data Genomics (BDG) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The BDG licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.bdgenomics.cannoli

import java.io.FileNotFoundException
import htsjdk.samtools.ValidationStringency
import org.apache.spark.SparkContext
import org.bdgenomics.adam.rdd.ADAMContext._
import org.bdgenomics.adam.rdd.read.{ AlignmentDataset, AnySAMOutFormatter }
import org.bdgenomics.adam.rdd.sequence.{ FASTAInFormatter, SequenceDataset }
import org.bdgenomics.adam.sql.{ Alignment => AlignmentProduct }
import org.bdgenomics.cannoli.builder.CommandBuilders
import org.bdgenomics.formats.avro.Alignment
import org.bdgenomics.utils.cli._
import org.kohsuke.args4j.{ Option => Args4jOption }
import scala.collection.JavaConversions._

/**
 * Blastn function arguments.
 */
class BlastnArgs extends Args4jBase {
  @Args4jOption(required = false, name = "-executable", usage = "Path to the blastn executable. Defaults to blastn.")
  var executable: String = "blastn"

  @Args4jOption(required = false, name = "-image", usage = "Container image to use. Defaults to quay.io/biocontainers/blast:2.9.0--pl526h3066fca_4.")
  var image: String = "quay.io/biocontainers/blast:2.9.0--pl526h3066fca_4"

  @Args4jOption(required = false, name = "-sudo", usage = "Run via sudo.")
  var sudo: Boolean = false

  @Args4jOption(required = false, name = "-add_files", usage = "If true, use the SparkFiles mechanism to distribute files to executors.")
  var addFiles: Boolean = false

  @Args4jOption(required = false, name = "-use_docker", usage = "If true, uses Docker to launch blastn.")
  var useDocker: Boolean = false

  @Args4jOption(required = false, name = "-use_singularity", usage = "If true, uses Singularity to launch blastn.")
  var useSingularity: Boolean = false

  @Args4jOption(required = true, name = "-db", usage = "BLAST database name.")
  var db: String = null

  @Args4jOption(required = false, name = "-evalue", usage = "Expectation value (E) threshold for saving hits.")
  var evalue: java.lang.Double = null

  @Args4jOption(required = false, name = "-word_size", usage = "Word size for wordfinder algorithm (length of best perfect match). If specified, must be at least 4.")
  var wordSize: java.lang.Integer = null

  @Args4jOption(required = false, name = "-gapopen", usage = "Cost to open a gap.")
  var gapOpen: java.lang.Integer = null

  @Args4jOption(required = false, name = "-gapextend", usage = "Cost to extend a gap.")
  var gapExtend: java.lang.Integer = null

  @Args4jOption(required = false, name = "-penalty", usage = "Penalty for a nucleotide mismatch. If specified, must be less than or equal to zero.")
  var penalty: java.lang.Integer = null

  @Args4jOption(required = false, name = "-reward", usage = "Reward for a nucleotide match. If specified, must be at least zero.")
  var reward: java.lang.Integer = null

  @Args4jOption(required = false, name = "-max_hsps", usage = "Maximum number of High-scoring Segment Pair (HSP)s per sequence.")
  var maxHsps: java.lang.Integer = null

  @Args4jOption(required = false, name = "-perc_identity", usage = "Percent identity.")
  var percentIdentity: java.lang.Double = null

  @Args4jOption(required = false, name = "-stringency", usage = "Stringency level for various checks; can be SILENT, LENIENT, or STRICT. Defaults to STRICT.")
  var stringency: String = "STRICT"
}

/**
 * Blastn wrapper as a function SequenceDataset &rarr; AlignmentDataset,
 * for use in cannoli-shell or notebooks.
 *
 * @param args Blastn function arguments.
 * @param sc Spark context.
 */
class Blastn(
    val args: BlastnArgs,
    sc: SparkContext) extends CannoliFn[SequenceDataset, AlignmentDataset](sc) {

  override def apply(sequences: SequenceDataset): AlignmentDataset = {

    // fail fast if db not found
    try {
      absolute(args.db)
    } catch {
      case e: FileNotFoundException => {
        error("BLAST database %s not found, touch an empty file at this path if it does not exist".format(args.db))
        throw e
      }
    }

    val builder = CommandBuilders.create(args.useDocker, args.useSingularity)
      .setExecutable(args.executable)
      .add("-db")
      .add(if (args.addFiles) "$0" else absolute(args.db))
      .add("-outfmt")
      .add("17 SR SQ")
      .add("-parse_deflines")

    Option(args.evalue).foreach(d => builder.add("-evalue").add(d.toString))
    Option(args.wordSize).foreach(i => builder.add("-word_size").add(i.toString))
    Option(args.gapOpen).foreach(i => builder.add("-gapopen").add(i.toString))
    Option(args.gapExtend).foreach(i => builder.add("-gapextend").add(i.toString))
    Option(args.penalty).foreach(i => builder.add("-penalty").add(i.toString))
    Option(args.reward).foreach(i => builder.add("-reward").add(i.toString))
    Option(args.maxHsps).foreach(i => builder.add("-max_hsps").add(i.toString))
    Option(args.percentIdentity).foreach(d => builder.add("-perc_identity").add(d.toString))

    if (args.addFiles) {
      // add args.db for "$0"
      builder.addFile(args.db)
      // add BLAST database files via globbed db path
      builder.addFiles(files(args.db + "*"))
    }

    if (args.useDocker || args.useSingularity) {
      builder
        .setImage(args.image)
        .setSudo(args.sudo)
        .addMount(if (args.addFiles) "$root" else root(args.db))
    }

    info("Piping %s to blastn with command: %s files: %s".format(
      sequences, builder.build(), builder.getFiles()))

    implicit val tFormatter = FASTAInFormatter
    implicit val uFormatter = new AnySAMOutFormatter(ValidationStringency.valueOf(args.stringency))<commit_msg>Add -num_threads argument to BlastnArgs.<commit_after>/**
 * Licensed to Big Data Genomics (BDG) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The BDG licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.bdgenomics.cannoli

import java.io.FileNotFoundException
import htsjdk.samtools.ValidationStringency
import org.apache.spark.SparkContext
import org.bdgenomics.adam.rdd.ADAMContext._
import org.bdgenomics.adam.rdd.read.{ AlignmentDataset, AnySAMOutFormatter }
import org.bdgenomics.adam.rdd.sequence.{ FASTAInFormatter, SequenceDataset }
import org.bdgenomics.adam.sql.{ Alignment => AlignmentProduct }
import org.bdgenomics.cannoli.builder.CommandBuilders
import org.bdgenomics.formats.avro.Alignment
import org.bdgenomics.utils.cli._
import org.kohsuke.args4j.{ Option => Args4jOption }
import scala.collection.JavaConversions._

/**
 * Blastn function arguments.
 */
class BlastnArgs extends Args4jBase {
  @Args4jOption(required = false, name = "-executable", usage = "Path to the blastn executable. Defaults to blastn.")
  var executable: String = "blastn"

  @Args4jOption(required = false, name = "-image", usage = "Container image to use. Defaults to quay.io/biocontainers/blast:2.9.0--pl526h3066fca_4.")
  var image: String = "quay.io/biocontainers/blast:2.9.0--pl526h3066fca_4"

  @Args4jOption(required = false, name = "-sudo", usage = "Run via sudo.")
  var sudo: Boolean = false

  @Args4jOption(required = false, name = "-add_files", usage = "If true, use the SparkFiles mechanism to distribute files to executors.")
  var addFiles: Boolean = false

  @Args4jOption(required = false, name = "-use_docker", usage = "If true, uses Docker to launch blastn.")
  var useDocker: Boolean = false

  @Args4jOption(required = false, name = "-use_singularity", usage = "If true, uses Singularity to launch blastn.")
  var useSingularity: Boolean = false

  @Args4jOption(required = true, name = "-db", usage = "BLAST database name.")
  var db: String = null

  @Args4jOption(required = false, name = "-evalue", usage = "Expectation value (E) threshold for saving hits.")
  var evalue: java.lang.Double = null

  @Args4jOption(required = false, name = "-word_size", usage = "Word size for wordfinder algorithm (length of best perfect match). If specified, must be at least 4.")
  var wordSize: java.lang.Integer = null

  @Args4jOption(required = false, name = "-gapopen", usage = "Cost to open a gap.")
  var gapOpen: java.lang.Integer = null

  @Args4jOption(required = false, name = "-gapextend", usage = "Cost to extend a gap.")
  var gapExtend: java.lang.Integer = null

  @Args4jOption(required = false, name = "-penalty", usage = "Penalty for a nucleotide mismatch. If specified, must be less than or equal to zero.")
  var penalty: java.lang.Integer = null

  @Args4jOption(required = false, name = "-reward", usage = "Reward for a nucleotide match. If specified, must be at least zero.")
  var reward: java.lang.Integer = null

  @Args4jOption(required = false, name = "-max_hsps", usage = "Maximum number of High-scoring Segment Pair (HSP)s per sequence.")
  var maxHsps: java.lang.Integer = null

  @Args4jOption(required = false, name = "-perc_identity", usage = "Percent identity.")
  var percentIdentity: java.lang.Double = null

  @Args4jOption(required = false, name = "-num_threads", usage = "Number of threads to use to run blastn. Defaults to 1.")
  var numThreads: java.lang.Integer = null

  @Args4jOption(required = false, name = "-stringency", usage = "Stringency level for various checks; can be SILENT, LENIENT, or STRICT. Defaults to STRICT.")
  var stringency: String = "STRICT"
}

/**
 * Blastn wrapper as a function SequenceDataset &rarr; AlignmentDataset,
 * for use in cannoli-shell or notebooks.
 *
 * @param args Blastn function arguments.
 * @param sc Spark context.
 */
class Blastn(
    val args: BlastnArgs,
    sc: SparkContext) extends CannoliFn[SequenceDataset, AlignmentDataset](sc) {

  override def apply(sequences: SequenceDataset): AlignmentDataset = {

    // fail fast if db not found
    try {
      absolute(args.db)
    } catch {
      case e: FileNotFoundException => {
        error("BLAST database %s not found, touch an empty file at this path if it does not exist".format(args.db))
        throw e
      }
    }

    val builder = CommandBuilders.create(args.useDocker, args.useSingularity)
      .setExecutable(args.executable)
      .add("-db")
      .add(if (args.addFiles) "$0" else absolute(args.db))
      .add("-outfmt")
      .add("17 SR SQ")
      .add("-parse_deflines")

    Option(args.evalue).foreach(d => builder.add("-evalue").add(d.toString))
    Option(args.wordSize).foreach(i => builder.add("-word_size").add(i.toString))
    Option(args.gapOpen).foreach(i => builder.add("-gapopen").add(i.toString))
    Option(args.gapExtend).foreach(i => builder.add("-gapextend").add(i.toString))
    Option(args.penalty).foreach(i => builder.add("-penalty").add(i.toString))
    Option(args.reward).foreach(i => builder.add("-reward").add(i.toString))
    Option(args.maxHsps).foreach(i => builder.add("-max_hsps").add(i.toString))
    Option(args.percentIdentity).foreach(d => builder.add("-perc_identity").add(d.toString))
    Option(args.numThreads).foreach(i => builder.add("-num_threads").add(i.toString))

    if (args.addFiles) {
      // add args.db for "$0"
      builder.addFile(args.db)
      // add BLAST database files via globbed db path
      builder.addFiles(files(args.db + "*"))
    }

    if (args.useDocker || args.useSingularity) {
      builder
        .setImage(args.image)
        .setSudo(args.sudo)
        .addMount(if (args.addFiles) "$root" else root(args.db))
    }

    info("Piping %s to blastn with command: %s files: %s".format(
      sequences, builder.build(), builder.getFiles()))

    implicit val tFormatter = FASTAInFormatter
    implicit val uFormatter = new AnySAMOutFormatter(ValidationStringency.valueOf(args.stringency))
{'fraction_non_alphanumeric': 0.12224806201550388, 'fraction_numerical': 0.007906976744186046, 'mean_word_length': 3.9391271056661563, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4936003', 'n_tokens_mistral': 4076, 'n_tokens_neox': 3938, 'n_words': 1181}