<commit_before>/* NOTE: Adapted from ScalaJSPartest.scala in
 * https://github.com/scala-js/scala-js/
 * TODO make partest configurable */

package dotty.partest

import scala.reflect.io.AbstractFile
import scala.tools.partest._
import scala.tools.partest.nest._
import TestState.{ Pass, Fail, Crash, Uninitialized, Updated }
import ClassPath.{ join, split }
import FileManager.{ compareFiles, compareContents, joinPaths, withTempFile }
import scala.util.matching.Regex
import tools.nsc.io.{ File => NSCFile }
import java.io.{ File, PrintStream, FileOutputStream, PrintWriter, FileWriter }
import java.net.URLClassLoader

/** Runs dotty partest from the Console, discovering test sources in
  * DPConfig.testRoot that have been generated automatically by
  * DPPrepJUnitRunner. Use `sbt partest` to run. If additional jars are
  * required by some run tests, add them to partestDeps in the sbt Build.scala.
  */
object DPConsoleRunner {
  def main(args: Array[String]): Unit = {
    // unfortunately sbt runTask passes args as single string
    // extra jars for run tests are passed with -dottyJars <count> <jar1> <jar2> ...
    val jarFinder = """-dottyJars (\d*) (.*)""".r
    val (jarList, otherArgs) = args.toList.partition(jarFinder.findFirstIn(_).isDefined)
    val (extraJars, moreArgs) = jarList match {
      case Nil => sys.error("Error: DPConsoleRunner needs \"-dottyJars <jarCount> <jars>*\".")
      case jarFinder(nr, jarString) :: Nil =>
        val jars = jarString.split(" ").toList
        val count = nr.toInt
        if (jars.length < count)
          sys.error("Error: DPConsoleRunner found wrong number of dottyJars: " + jars + ", expected: " + nr)
        else (jars.take(count), jars.drop(count))
      case list => sys.error("Error: DPConsoleRunner found several -dottyJars options: " + list)
    }
    new DPConsoleRunner((otherArgs ::: moreArgs) mkString (" "), extraJars).runPartest
  }
}

// console runner has a suite runner which creates a test runner for each test
class DPConsoleRunner(args: String, extraJars: List[String]) extends ConsoleRunner(args) {
  override val suiteRunner = new DPSuiteRunner (
    testSourcePath = optSourcePath getOrElse DPConfig.testRoot,
    fileManager = new DottyFileManager(extraJars),
    updateCheck = optUpdateCheck,
    failed = optFailed,
    consoleArgs = args)

  override def run = {}
  def runPartest = super.run
}

class DottyFileManager(extraJars: List[String]) extends FileManager(Nil) {
  lazy val extraJarList = extraJars.map(NSCFile(_))
  override lazy val libraryUnderTest  = Path(extraJars.find(_.contains("scala-library")).getOrElse(""))
  override lazy val reflectUnderTest  = Path(extraJars.find(_.contains("scala-reflect")).getOrElse(""))
  override lazy val compilerUnderTest = Path(extraJars.find(_.contains("dotty")).getOrElse(""))
}

class DPSuiteRunner(testSourcePath: String, // relative path, like "files", or "pending"
  fileManager: DottyFileManager,
  updateCheck: Boolean,
  failed: Boolean,
  consoleArgs: String,
  javaCmdPath: String = PartestDefaults.javaCmd,
  javacCmdPath: String = PartestDefaults.javacCmd,
  scalacExtraArgs: Seq[String] = Seq.empty,
  javaOpts: String = DPConfig.runJVMOpts)
extends SuiteRunner(testSourcePath, fileManager, updateCheck, failed, javaCmdPath, javacCmdPath, scalacExtraArgs, javaOpts) {

  if (!DPConfig.runTestsInParallel)
    sys.props("partest.threads") = "1"

  sys.props("partest.root") = "."

  // override to provide Dotty banner
  override def banner: String = {
    s"""|Welcome to Partest for Dotty! Partest version: ${Properties.versionNumberString}
        |Compiler under test: dotty.tools.dotc.Bench or dotty.tools.dotc.Main
        |Generated test sources: ${PathSettings.srcDir}${File.separator}
        |Test directories: ${DPConfig.testDirs.toList.mkString(", ")}
        |Debugging: failed tests have compiler output in test-kind.clog, run output in test-kind.log, class files in test-kind.obj
        |Parallel: ${DPConfig.runTestsInParallel}
        |Options: (use partest --help for usage information) ${consoleArgs}
    """.stripMargin
  }

  // override for DPTestRunner and redirecting compilation output to test.clog
  override def runTest(testFile: File): TestState = {
    val runner = new DPTestRunner(testFile, this)

    val state =
      try {
        runner.run match {<commit_msg>Run some tests sequentially to avoid exhausting available memory<commit_after>/* NOTE: Adapted from ScalaJSPartest.scala in
 * https://github.com/scala-js/scala-js/
 * TODO make partest configurable */

package dotty.partest

import scala.reflect.io.AbstractFile
import scala.tools.partest._
import scala.tools.partest.nest._
import TestState.{ Pass, Fail, Crash, Uninitialized, Updated }
import ClassPath.{ join, split }
import FileManager.{ compareFiles, compareContents, joinPaths, withTempFile }
import scala.util.matching.Regex
import tools.nsc.io.{ File => NSCFile }
import java.io.{ File, PrintStream, FileOutputStream, PrintWriter, FileWriter }
import java.net.URLClassLoader

/** Runs dotty partest from the Console, discovering test sources in
  * DPConfig.testRoot that have been generated automatically by
  * DPPrepJUnitRunner. Use `sbt partest` to run. If additional jars are
  * required by some run tests, add them to partestDeps in the sbt Build.scala.
  */
object DPConsoleRunner {
  def main(args: Array[String]): Unit = {
    // unfortunately sbt runTask passes args as single string
    // extra jars for run tests are passed with -dottyJars <count> <jar1> <jar2> ...
    val jarFinder = """-dottyJars (\d*) (.*)""".r
    val (jarList, otherArgs) = args.toList.partition(jarFinder.findFirstIn(_).isDefined)
    val (extraJars, moreArgs) = jarList match {
      case Nil => sys.error("Error: DPConsoleRunner needs \"-dottyJars <jarCount> <jars>*\".")
      case jarFinder(nr, jarString) :: Nil =>
        val jars = jarString.split(" ").toList
        val count = nr.toInt
        if (jars.length < count)
          sys.error("Error: DPConsoleRunner found wrong number of dottyJars: " + jars + ", expected: " + nr)
        else (jars.take(count), jars.drop(count))
      case list => sys.error("Error: DPConsoleRunner found several -dottyJars options: " + list)
    }
    new DPConsoleRunner((otherArgs ::: moreArgs) mkString (" "), extraJars).runPartest
  }
}

// console runner has a suite runner which creates a test runner for each test
class DPConsoleRunner(args: String, extraJars: List[String]) extends ConsoleRunner(args) {
  override val suiteRunner = new DPSuiteRunner (
    testSourcePath = optSourcePath getOrElse DPConfig.testRoot,
    fileManager = new DottyFileManager(extraJars),
    updateCheck = optUpdateCheck,
    failed = optFailed,
    consoleArgs = args)

  override def run = {}
  def runPartest = super.run
}

class DottyFileManager(extraJars: List[String]) extends FileManager(Nil) {
  lazy val extraJarList = extraJars.map(NSCFile(_))
  override lazy val libraryUnderTest  = Path(extraJars.find(_.contains("scala-library")).getOrElse(""))
  override lazy val reflectUnderTest  = Path(extraJars.find(_.contains("scala-reflect")).getOrElse(""))
  override lazy val compilerUnderTest = Path(extraJars.find(_.contains("dotty")).getOrElse(""))
}

class DPSuiteRunner(testSourcePath: String, // relative path, like "files", or "pending"
  fileManager: DottyFileManager,
  updateCheck: Boolean,
  failed: Boolean,
  consoleArgs: String,
  javaCmdPath: String = PartestDefaults.javaCmd,
  javacCmdPath: String = PartestDefaults.javacCmd,
  scalacExtraArgs: Seq[String] = Seq.empty,
  javaOpts: String = DPConfig.runJVMOpts)
extends SuiteRunner(testSourcePath, fileManager, updateCheck, failed, javaCmdPath, javacCmdPath, scalacExtraArgs, javaOpts) {

  if (!DPConfig.runTestsInParallel)
    sys.props("partest.threads") = "1"

  sys.props("partest.root") = "."

  // override to provide Dotty banner
  override def banner: String = {
    s"""|Welcome to Partest for Dotty! Partest version: ${Properties.versionNumberString}
        |Compiler under test: dotty.tools.dotc.Bench or dotty.tools.dotc.Main
        |Generated test sources: ${PathSettings.srcDir}${File.separator}
        |Test directories: ${DPConfig.testDirs.toList.mkString(", ")}
        |Debugging: failed tests have compiler output in test-kind.clog, run output in test-kind.log, class files in test-kind.obj
        |Parallel: ${DPConfig.runTestsInParallel}
        |Options: (use partest --help for usage information) ${consoleArgs}
    """.stripMargin
  }

  /** Tests which are compiled with one or more of the flags in this list will be run
   *  one by one, without any other test running at the same time.
   *  This is necessary because some test flags require a lot of memory when running
   *  the compiler and may exhaust the available memory when run in parallel with other tests.
   */
  def sequentialFlags = List("-Ytest-pickler")

  override def runTestsForFiles(kindFiles: Array[File], kind: String): Array[TestState] = {
    val (sequentialTests, parallelTests) =
      kindFiles partition { kindFile =>
        val flags = kindFile.changeExtension("flags").fileContents
        sequentialFlags.exists(seqFlag => flags.contains(seqFlag))
      }

    val seqResults =
      if (!sequentialTests.isEmpty) {
        val savedThreads = sys.props("partest.threads")
        sys.props("partest.threads") = "1"

        NestUI.echo(s"## we will run ${sequentialTests.length} tests sequentially")
        val res = super.runTestsForFiles(sequentialTests, kind)

        if (savedThreads != null)
          sys.props("partest.threads") = savedThreads
        else
          sys.props.remove("partest.threads")

        res
      } else Array[TestState]()

    val parResults =
      if (!parallelTests.isEmpty) {
        NestUI.echo(s"## we will run ${parallelTests.length} tests in parallel using ${PartestDefaults.numThreads} threads")
        super.runTestsForFiles(parallelTests, kind)
      } else Array[TestState]()

    seqResults ++ parResults
  }

  // override for DPTestRunner and redirecting compilation output to test.clog
  override def runTest(testFile: File): TestState = {
    val runner = new DPTestRunner(testFile, this)

    val state =
      try {
        runner.run match {
{'fraction_non_alphanumeric': 0.11019095869056898, 'fraction_numerical': 0.0006819953234606392, 'mean_word_length': 3.932724651609803, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 23, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4223105', 'n_tokens_mistral': 3156, 'n_tokens_neox': 3060, 'n_words': 884}