<commit_before>package water.sparkling.itest

import org.apache.spark.deploy.SparkSubmit
import org.scalatest.{BeforeAndAfterEach, Tag, Suite}
import scala.collection.mutable

/**
 * Integration test support to be run on top of Spark.
 */
trait SparkITest extends BeforeAndAfterEach { self: Suite =>

  private var testEnv:IntegTestEnv = _

  override protected def beforeEach(): Unit = {
    super.beforeEach()
    testEnv = new TestEnvironment
  }

  override protected def afterEach(): Unit = {
    testEnv = null
    super.afterEach()
  }

  /** Launch given class name via SparkSubmit and use given environment
    * to configure SparkSubmit command line.
    * @param className name of class to launch as integration test
    * @param env Spark environment
    */
  def launch(className: String, env: IntegTestEnv): Unit = {
    val cmdLine = Seq[String](
      "--class", className,
      "--jars", env.swassembly,
      "--verbose",
      "--master", env.sparkMaster) ++
    // Disable GA collection by default
    env.sparkConf.flatMap( p => Seq("--conf", s"${p._1}=${p._2}") ) ++
                              Seq("--conf", "spark.ext.h2o.disable.ga=true") ++
                              Seq[String](env.testJar)

    if(!env.sparkMaster.startsWith("yarn")) {
      SparkSubmit.main(cmdLine.toArray)
    } else {
      // Launch it via command line
      import scala.sys.process._
      val sparkHome = System.getenv("SPARK_HOME")
      val cmdToLaunch = Seq( s"$sparkHome/bin/spark-submit") ++ cmdLine
      val proc = cmdToLaunch.!
      assert (proc == 0, "Process finished in wrong way!")
    }
  }

  object env {
    def apply(init: => Unit):IntegTestEnv = {
      val e = testEnv
      val result = init
      e
    }
  }

  trait IntegTestEnv {
    lazy val swassembly = sys.props.getOrElse("sparkling.test.assembly",
      fail("The variable 'sparkling.test.assembly' is not set! It should point to assembly jar file."))

    lazy val testJar = sys.props.getOrElse("sparkling.test.jar",
      fail("The variable 'sparkling.test.jar' should point to a jar containing this file!"))

    lazy val sparkMaster = sys.props.getOrElse("MASTER",
      sys.props.getOrElse("spark.master",
        fail("The variable 'MASTER' should point to Spark cluster")))<commit_msg>Transform spark.driver.memory to explicit option '--driver-memory'<commit_after>package water.sparkling.itest

import org.apache.spark.deploy.SparkSubmit
import org.scalatest.{BeforeAndAfterEach, Tag, Suite}
import scala.collection.mutable

/**
 * Integration test support to be run on top of Spark.
 */
trait SparkITest extends BeforeAndAfterEach { self: Suite =>

  private var testEnv:IntegTestEnv = _

  override protected def beforeEach(): Unit = {
    super.beforeEach()
    testEnv = new TestEnvironment
  }

  override protected def afterEach(): Unit = {
    testEnv = null
    super.afterEach()
  }

  /** Launch given class name via SparkSubmit and use given environment
    * to configure SparkSubmit command line.
    * @param className name of class to launch as integration test
    * @param env Spark environment
    */
  def launch(className: String, env: IntegTestEnv): Unit = {
    val cmdLine = Seq[String](
      "--class", className,
      "--jars", env.swassembly,
      "--verbose",
      "--master", env.sparkMaster) ++
      env.sparkConf.get("spark.driver.memory").map(m => Seq("--driver-memory", m)).getOrElse(Nil) ++
      // Disable GA collection by default
      env.sparkConf.flatMap( p => Seq("--conf", s"${p._1}=${p._2}") ) ++
      Seq("--conf", "spark.ext.h2o.disable.ga=true") ++
      Seq[String](env.testJar)

    if(!env.sparkMaster.startsWith("yarn")) {
      SparkSubmit.main(cmdLine.toArray)
    } else {
      // Launch it via command line
      import scala.sys.process._
      val sparkHome = System.getenv("SPARK_HOME")
      val cmdToLaunch = Seq( s"$sparkHome/bin/spark-submit") ++ cmdLine
      val proc = cmdToLaunch.!
      assert (proc == 0, "Process finished in wrong way!")
    }
  }

  object env {
    def apply(init: => Unit):IntegTestEnv = {
      val e = testEnv
      val result = init
      e
    }
  }

  trait IntegTestEnv {
    lazy val swassembly = sys.props.getOrElse("sparkling.test.assembly",
      fail("The variable 'sparkling.test.assembly' is not set! It should point to assembly jar file."))

    lazy val testJar = sys.props.getOrElse("sparkling.test.jar",
      fail("The variable 'sparkling.test.jar' should point to a jar containing this file!"))

    lazy val sparkMaster = sys.props.getOrElse("MASTER",
      sys.props.getOrElse("spark.master",
        fail("The variable 'MASTER' should point to Spark cluster")))
{'fraction_non_alphanumeric': 0.12467921300256629, 'fraction_numerical': 0.001710863986313088, 'mean_word_length': 2.953508030431107, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6126060', 'n_tokens_mistral': 1480, 'n_tokens_neox': 1406, 'n_words': 389}