<commit_before>package se.uu.farmbio.vs.examples

import java.io.PrintWriter

import org.apache.spark.Logging
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext

import openeye.oedocking.OEDockMethod
import openeye.oedocking.OESearchResolution
import scopt.OptionParser
import se.uu.farmbio.vs.SBVSPipeline

object Docker extends Logging {
  case class Params(
    master: String = null,
    conformersFile: String = null,
    receptorFile: String = null,
    topPosesPath: String = null,
    size: String = "30",
    sampleSize: Double = 1.0,
    collapse: Int = 0)

  def main(args: Array[String]) {

    val defaultParams = Params()

    val parser = new OptionParser[Params]("Docker") {
      head("Docker: an example docking pipeline.")
      opt[String]("collapse")
        .text("number of best scoring molecules with same id returned (default: all of them).")
        .action((x, c) => c.copy(collapse = x.toInt))
      opt[String]("size")
        .text("it controls how many molecules are handled within a task (default: 30).")
        .action((x, c) => c.copy(size = x))
      opt[String]("sampleSize")
        .text("it reduces the input size to the specified fraction (default: 1.0, means no reduction). " +
          "It can be used to evaluate scalability.")
        .action((x, c) => c.copy(sampleSize = x.toDouble))
      opt[String]("master")
        .text("spark master")
        .action((x, c) => c.copy(master = x))
      arg[String]("<conformers-file>")
        .required()
        .text("path to input SDF conformers file")
        .action((x, c) => c.copy(conformersFile = x))
      arg[String]("<receptor-file>")
        .required()
        .text("path to input OEB receptor file")
        .action((x, c) => c.copy(receptorFile = x))
      arg[String]("<top-poses-path>")
        .required()
        .text("path to top output poses")
        .action((x, c) => c.copy(topPosesPath = x))
    }

    parser.parse(args, defaultParams).map { params =>
      run(params)
    } getOrElse {
      sys.exit(1)
    }

  }

  def run(params: Params) {

    //Init Spark
    val conf = new SparkConf()
      .setAppName("Docker")
      .setExecutorEnv("OE_LICENSE", System.getenv("OE_LICENSE"))
    if (params.master != null) {
      conf.setMaster(params.master)
    }
    val sc = new SparkContext(conf)
    sc.hadoopConfiguration.set("se.uu.farmbio.parsers.SDFRecordReader.size", params.size)

    val t0 = System.currentTimeMillis
    var sampleRDD = new SBVSPipeline(sc)
      .readConformerFile(params.conformersFile)
      .getMolecules

    if (params.sampleSize < 1.0) { //Samples Data on the basis of sampleSize Parameter
      sampleRDD = sampleRDD.sample(false, params.sampleSize) //Does not take effect for complete set
    }

    var poses = new SBVSPipeline(sc)
      .readConformerRDDs(Seq(sampleRDD))
      .dock(params.receptorFile, OEDockMethod.Chemgauss4, OESearchResolution.Standard)
    if (params.collapse > 0) {
      poses = poses.collapse(params.collapse)
    }
    val res = poses
      .sortByScore
      .getMolecules
      .take(10) //take first 10
    val t1 = System.currentTimeMillis
    val elapsed = t1 - t0<commit_msg>Save all output posses to HDFS<commit_after>package se.uu.farmbio.vs.examples

import java.io.PrintWriter

import org.apache.spark.Logging
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext

import openeye.oedocking.OEDockMethod
import openeye.oedocking.OESearchResolution
import scopt.OptionParser
import se.uu.farmbio.vs.SBVSPipeline

object Docker extends Logging {
  case class Params(
    master: String = null,
    conformersFile: String = null,
    receptorFile: String = null,
    topPosesPath: String = null,
    size: String = "30",
    sampleSize: Double = 1.0,
    collapse: Int = 0,
    hdfsPath: String = null)

  def main(args: Array[String]) {

    val defaultParams = Params()

    val parser = new OptionParser[Params]("Docker") {
      head("Docker: an example docking pipeline.")
      opt[String]("collapse")
        .text("number of best scoring molecules with same id returned (default: all of them).")
        .action((x, c) => c.copy(collapse = x.toInt))
      opt[String]("size")
        .text("it controls how many molecules are handled within a task (default: 30).")
        .action((x, c) => c.copy(size = x))
      opt[String]("sampleSize")
        .text("it reduces the input size to the specified fraction (default: 1.0, means no reduction). " +
          "It can be used to evaluate scalability.")
        .action((x, c) => c.copy(sampleSize = x.toDouble))
      opt[String]("master")
        .text("spark master")
        .action((x, c) => c.copy(master = x))
      arg[String]("<conformers-file>")
        .required()
        .text("path to input SDF conformers file")
        .action((x, c) => c.copy(conformersFile = x))
      arg[String]("<receptor-file>")
        .required()
        .text("path to input OEB receptor file")
        .action((x, c) => c.copy(receptorFile = x))
      arg[String]("<top-poses-path>")
        .required()
        .text("path to top output poses")
        .action((x, c) => c.copy(topPosesPath = x))
      arg[String]("<hdfs-path>")
        .required()
        .text("path to save all output poses to hdfs")
        .action((x, c) => c.copy(hdfsPath = x))
    }

    parser.parse(args, defaultParams).map { params =>
      run(params)
    } getOrElse {
      sys.exit(1)
    }

  }

  def run(params: Params) {

    //Init Spark
    val conf = new SparkConf()
      .setAppName("Docker")
      .setExecutorEnv("OE_LICENSE", System.getenv("OE_LICENSE"))
    if (params.master != null) {
      conf.setMaster(params.master)
    }
    val sc = new SparkContext(conf)
    sc.hadoopConfiguration.set("se.uu.farmbio.parsers.SDFRecordReader.size", params.size)

    val t0 = System.currentTimeMillis
    var sampleRDD = new SBVSPipeline(sc)
      .readConformerFile(params.conformersFile)
      .getMolecules

    if (params.sampleSize < 1.0) { //Samples Data on the basis of sampleSize Parameter
      sampleRDD = sampleRDD.sample(false, params.sampleSize) //Does not take effect for complete set
    }

    var poses = new SBVSPipeline(sc)
      .readConformerRDDs(Seq(sampleRDD))
      .dock(params.receptorFile, OEDockMethod.Chemgauss4, OESearchResolution.Standard)
    if (params.collapse > 0) {
      poses = poses.collapse(params.collapse)
    }
    val res = poses
      .sortByScore
      .saveAsTextFile(params.hdfsPath)
      .getMolecules
      .take(10) //take first 10
    val t1 = System.currentTimeMillis
    val elapsed = t1 - t0
{'fraction_non_alphanumeric': 0.1300468773627703, 'fraction_numerical': 0.00665356116739755, 'mean_word_length': 2.8187066974595845, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2653189', 'n_tokens_mistral': 2202, 'n_tokens_neox': 2110, 'n_words': 524}