<commit_before>package com.tribbloids.spookystuff.session

import com.tribbloids.spookystuff.actions.PyAction
import com.tribbloids.spookystuff.{PythonException, SpookyEnvFixture}

/**
  * Created by peng on 01/08/16.
  */
object PythonDriverSuite {

  def send1PlusX(xs: Seq[Int]): Unit = {
    runIterable(xs) {
      (i, proc) =>
        val r = proc.sendAndGetResult(s"print($i + 1)")
        assert(r.replace(">>> ", "").trim == (i + 1).toString)
    }
  }

  def runIterable[T, R](xs: Iterable[T])(f: (T, PythonDriver) => R): Iterable[R] = {
    val proc = new PythonDriver("python")
    proc.open()
    try {
      val result = xs.map{
        f(_, proc)
      }
      result
    }
    finally {
      proc.close()
    }
  }
}

class PythonDriverSuite extends SpookyEnvFixture {

  test("sendAndGetResult should work in single thread") {
    PythonDriverSuite.send1PlusX(1 to 100)
  }

  test("sendAndGetResult should work in multiple threads") {
    val rdd = sc.parallelize(1 to 100)
    assert(rdd.getNumPartitions > 1)
    rdd.foreachPartition{
      it =>
        val seq = it.toSeq
        PythonDriverSuite.send1PlusX(seq)
    }
  }

  test("sendAndGetResult should work if interpretation triggers an error") {
    PythonDriverSuite.runIterable(1 to 10) {
      (i, proc) =>
        val r = proc.sendAndGetResult(s"print($i / 0)")
        assert(r.replace(">>> ", "").trim startsWith "Traceback")
    }
  }

  test("interpret should yield 1 row for a single print") {

    PythonDriverSuite.runIterable(1 to 10) {
      (i, proc) =>
        val result = proc.interpret(s"print($i * $i)").mkString("\n")
        result.shouldBe(
          "" + i*i
        )
    }
  }

  test("interpret should throw an exception if interpreter raises an error") {

    PythonDriverSuite.runIterable(1 to 10) {
      (i, proc) =>
        intercept[PythonException]{
          proc.interpret(s"print($i / 0)")
        }
    }
  }


  test("interpret should throw an exception if interpreter raises a multi-line error") {

    PythonDriverSuite.runIterable(1 to 10) {
      (i, proc) =>
//        intercept[PythonException]{
          val result = proc.interpret(
            s"""
              |raise Exception(
              |${PyAction.QQQ}
              |abc
              |def
              |ghi
              |jkl
              |${PyAction.QQQ}
              |)
            """.stripMargin
          )
//        }
        result.foreach(println)
    }
  }
}<commit_msg>fix a PythonDriver test error<commit_after>package com.tribbloids.spookystuff.session

import com.tribbloids.spookystuff.actions.PyAction
import com.tribbloids.spookystuff.{PythonException, SpookyEnvFixture}

/**
  * Created by peng on 01/08/16.
  */
object PythonDriverSuite {

  def send1PlusX(xs: Seq[Int]): Unit = {
    runIterable(xs) {
      (i, proc) =>
        val r = proc.sendAndGetResult(s"print($i + 1)")
        assert(r.replace(">>> ", "").trim == (i + 1).toString)
    }
  }

  def runIterable[T, R](xs: Iterable[T])(f: (T, PythonDriver) => R): Iterable[R] = {
    val proc = new PythonDriver("python")
    proc.open()
    try {
      val result = xs.map{
        f(_, proc)
      }
      result
    }
    finally {
      proc.close()
    }
  }
}

class PythonDriverSuite extends SpookyEnvFixture {

  test("sendAndGetResult should work in single thread") {
    PythonDriverSuite.send1PlusX(1 to 100)
  }

  test("sendAndGetResult should work in multiple threads") {
    val rdd = sc.parallelize(1 to 100)
    assert(rdd.getNumPartitions > 1)
    rdd.foreachPartition{
      it =>
        val seq = it.toSeq
        PythonDriverSuite.send1PlusX(seq)
    }
  }

  test("sendAndGetResult should work if interpretation triggers an error") {
    PythonDriverSuite.runIterable(1 to 10) {
      (i, proc) =>
        val r = proc.sendAndGetResult(s"print($i / 0)")
        assert(r.replace(">>> ", "").trim startsWith "Traceback")
    }
  }

  test("interpret should yield 1 row for a single print") {

    PythonDriverSuite.runIterable(1 to 10) {
      (i, proc) =>
        val result = proc.interpret(s"print($i * $i)").mkString("\n")
        result.shouldBe(
          "" + i*i
        )
    }
  }

  test("interpret should throw an exception if interpreter raises an error") {

    PythonDriverSuite.runIterable(1 to 10) {
      (i, proc) =>
        intercept[PythonException]{
          proc.interpret(s"print($i / 0)")
        }
    }
  }


  test("interpret should throw an exception if interpreter raises a multi-line error") {

    PythonDriverSuite.runIterable(1 to 10) {
      (i, proc) =>
        intercept[PythonException]{
          val result = proc.interpret(
            s"""
               |raise Exception(
               |${PyAction.QQQ}
               |abc
               |def
               |ghi
               |jkl
               |${PyAction.QQQ}
               |)
            """.stripMargin
          )
        }
    }
  }
}
{'fraction_non_alphanumeric': 0.1293138449045879, 'fraction_numerical': 0.014210312626877792, 'mean_word_length': 2.1322314049586777, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1119080', 'n_tokens_mistral': 1668, 'n_tokens_neox': 1580, 'n_words': 349}