<commit_before>package edu.nus.systemtesting.hipsleek.app

import java.io.File
import java.io.PrintWriter
import java.nio.file.Path
import java.nio.file.Paths
import scala.io.Source
import scala.concurrent.Channel
import edu.nus.systemtesting.ExpectsOutput
import edu.nus.systemtesting.PreparedSystem
import edu.nus.systemtesting.Testable
import edu.nus.systemtesting.TestCase
import edu.nus.systemtesting.TestCaseConfiguration
import edu.nus.systemtesting.TestCaseBuilder
import edu.nus.systemtesting.TestCaseResult
import edu.nus.systemtesting.hg.Commit
import edu.nus.systemtesting.hg.Repository
import edu.nus.systemtesting.output.GlobalReporter.reporter
import edu.nus.systemtesting.hipsleek.BuildResult
import edu.nus.systemtesting.hipsleek.HipTestCase
import edu.nus.systemtesting.hipsleek.HipTestSuiteUsage
import edu.nus.systemtesting.hipsleek.SleekTestCase
import edu.nus.systemtesting.hipsleek.SleekTestSuiteUsage
import edu.nus.systemtesting.hipsleek.ValidateableSleekTestCase
import edu.nus.systemtesting.hipsleek.SuccessfulBuildResult
import edu.nus.systemtesting.serialisation.ResultsArchive
import edu.nus.systemtesting.testsuite.TestSuiteComparison
import edu.nus.systemtesting.testsuite.TestSuiteResult

/**
 * Generates, runs a subset of some set of testables.
 *
 * Does not deal with legacy, misnamed `run-fast-tests.pl`.
 */
class RunFast(config: AppConfig) {
  val repoDirPath = config.repoDirOrDie
  val repoDir = repoDirPath.toFile

  // Each instance of `ConfiguredMain` only ever uses the one `Repository`
  val repo = new Repository(config.repoDirOrDie)


  val validate = new Validate(config)


  def DefaultFastCacheName(name: String) = s".fasttests_$name"

  // might make sense to use SQLite db here.

  private case class FastTestRow(filename: String, args: Option[String] = None) {
    override def toString(): String =
      filename + (args map (a => " " + a) getOrElse "")
  }

  private def loadFastTests(file: File): Option[List[FastTestRow]] = {
    if (file.exists) {
      val src = Source fromFile file
      val content = src.mkString
      src.close()

      Some(content.lines.toList map { line =>
        line.split(" ", 2) match {
          case Array(filename) => FastTestRow(filename)
          case Array(filename, args) => FastTestRow(filename, Some(args))
        }
      })
    } else {
      None
    }
  }

  private def saveToFile(file: File, rows: List[FastTestRow]): Unit = {
    val out = new PrintWriter(file)

    rows foreach { row =>
      out.println(row.toString())
    }

    out.flush()
    out.close()
  }

  private def fileForSuite(suite: Suite): File =
    new File(DefaultFastCacheName(suite.toString()))



  private def args(arguments: String): Option[String] =
    if (arguments == "") None else Some(arguments)

  private def rowFromTestable(tc: Testable): FastTestRow = {
    import tc.{ fileName, arguments }

    FastTestRow(fileName.toString(), args(arguments))
  }

  private def filterTestable(all: List[Testable with ExpectsOutput],
                             loaded: List[FastTestRow]): List[Testable with ExpectsOutput] = {
    val loadedSet = loaded.toSet

    all filter { tc => loadedSet contains rowFromTestable(tc) }
  }

  private def loadFastTests(suite: Suite): Option[List[Testable with ExpectsOutput]] = {
    loadFastTests(fileForSuite(suite)) map { fastTests =>
      suite match {
        case HipOnly()   => filterTestable(HipTestSuiteUsage.allTestable, fastTests)
        case SleekOnly() => filterTestable(SleekTestSuiteUsage.allTestable, fastTests)

        // This is a little more dubious,
        // since typically Sleek+Hip are treated distinctly.
        case All() => filterTestable(SleekTestSuiteUsage.allTestable ++ HipTestSuiteUsage.allTestable, fastTests)

        case SleekValidateOnly() => {
          // construct ValidateableSleekTestCase directly
          fastTests map { fastTest =>
            TestCaseBuilder(Paths.get("sleek"), Paths.get(fastTest.filename))
          }
        }
      }
    }
  }

  private def saveFastTests(suite: Suite, data: List[Testable]): Unit = {
    saveToFile(fileForSuite(suite), data map rowFromTestable)
  }

  private def allConstructTestCase(ps: PreparedSystem, tc: Testable with ExpectsOutput, conf: TestCaseConfiguration):
      TestCase = {
    if (tc.toString endsWith "hip") {
      HipTestCase.constructTestCase(ps, tc, conf)
    } else {
      // assume if-not-hip then must be sleek
      SleekTestCase.constructTestCase(ps, tc, conf)
    }
  }

  private def generateFastTestablesForSuite(suite: Suite): List[Testable] = {
    // get the universe of testable for the suite,
    val allTestable = suite match {
      case HipOnly()           => HipTestSuiteUsage.allTestable
      case SleekOnly()         => SleekTestSuiteUsage.allTestable
      case All()               => SleekTestSuiteUsage.allTestable ++ HipTestSuiteUsage.allTestable
      case SleekValidateOnly() => validate.allTestable
    }

    val construct: (PreparedSystem, Testable with ExpectsOutput, TestCaseConfiguration) => TestCase =
      suite match {
        case HipOnly()           => HipTestCase.constructTestCase
        case SleekOnly()         => SleekTestCase.constructTestCase

        // This is more involved;
        case All()               => allConstructTestCase
        case SleekValidateOnly() => ValidateableSleekTestCase.constructTestCase
      }

    // if we have sufficent number of results, can compute from that,
    val resArch = new ResultsArchive(config.resultsDir, config.buildFailuresFile)
    val extantResults = allTestable map (resArch.resultsFor) filterNot (_.isEmpty)

    val timeTestablePairs = if (extantResults.length == allTestable.length) {
      extantResults map { res =>
        val timings = res map { case (rev, tcr) => tcr.executionTime }
        val avgTiming = timings.foldLeft(0L)({ (sum, time) => sum + time }) / timings.length

        val testable = res.head._2

        (avgTiming, testable)
      }
    } else {
      // otherwise, must run with a short timeout + don't save results (for T/O),
      // so as to see which tests are "quick"

      // Only save results for tests which take 10s or less.
      val QuickTimeout = 10
      val quickConfig = config.copy(saveResultOnTimeout = false,
                                    timeout = QuickTimeout)

      val runHipSleek = new RunHipSleek(quickConfig)
      import runHipSleek.{ altRunTests, runTestCaseForRevision }

      // If repo is dirty, this will be needlessly expensive.
      val repoC = repo.identify()

      // n.b. *might* throw UnableToBuildException here
      // XXX BUG: It seems this doesn't save the test case result, even for non-timeout?
      val tsr = altRunTests(construct, allTestable)(repoC)

      // wait for the results
      tsr.results map { tcr => (tcr.executionTime, tcr) }
    }

    val sortedTCRs = timeTestablePairs.sortBy { case (time, tc) => time }

    // Try to keep the timing under 2 mins for running the tests
    val FastTestTime = 120 * 1000

    sortedTCRs.foldLeft(List[TestCaseResult]())({ (fastTests, timeTCPair) =>
      val (tcTime, tc) = timeTCPair

      val totalTime = fastTests.foldLeft(0L) { (sum, tcr) => sum + tcr.executionTime }

      if (totalTime < FastTestTime) {
        fastTests :+ tc
      } else {
        fastTests
      }
    })
  }

  private def suiteFromString(suite: String): Suite =
    suite match {
      case "hip" => HipOnly()
      case "sleek" => SleekOnly()
      case "all" => All()
      case "sleek-validate" => SleekValidateOnly()
      case "validate-sleek" => SleekValidateOnly()
      case _ => throw new IllegalArgumentException(s"Unknown suite: $suite")
    }

  def run(): Unit = {<commit_msg>RunFast now only needs sufficient # of results, not EVERY result.<commit_after>package edu.nus.systemtesting.hipsleek.app

import java.io.File
import java.io.PrintWriter
import java.nio.file.Path
import java.nio.file.Paths
import scala.io.Source
import scala.concurrent.Channel
import edu.nus.systemtesting.ExpectsOutput
import edu.nus.systemtesting.PreparedSystem
import edu.nus.systemtesting.Testable
import edu.nus.systemtesting.TestCase
import edu.nus.systemtesting.TestCaseConfiguration
import edu.nus.systemtesting.TestCaseBuilder
import edu.nus.systemtesting.TestCaseResult
import edu.nus.systemtesting.hg.Commit
import edu.nus.systemtesting.hg.Repository
import edu.nus.systemtesting.output.GlobalReporter.reporter
import edu.nus.systemtesting.hipsleek.BuildResult
import edu.nus.systemtesting.hipsleek.HipTestCase
import edu.nus.systemtesting.hipsleek.HipTestSuiteUsage
import edu.nus.systemtesting.hipsleek.SleekTestCase
import edu.nus.systemtesting.hipsleek.SleekTestSuiteUsage
import edu.nus.systemtesting.hipsleek.ValidateableSleekTestCase
import edu.nus.systemtesting.hipsleek.SuccessfulBuildResult
import edu.nus.systemtesting.serialisation.ResultsArchive
import edu.nus.systemtesting.testsuite.TestSuiteComparison
import edu.nus.systemtesting.testsuite.TestSuiteResult

/**
 * Generates, runs a subset of some set of testables.
 *
 * Does not deal with legacy, misnamed `run-fast-tests.pl`.
 */
class RunFast(config: AppConfig) {
  val repoDirPath = config.repoDirOrDie
  val repoDir = repoDirPath.toFile

  // Each instance of `ConfiguredMain` only ever uses the one `Repository`
  val repo = new Repository(config.repoDirOrDie)


  val validate = new Validate(config)


  def DefaultFastCacheName(name: String) = s".fasttests_$name"

  // might make sense to use SQLite db here.

  private case class FastTestRow(filename: String, args: Option[String] = None) {
    override def toString(): String =
      filename + (args map (a => " " + a) getOrElse "")
  }

  private def loadFastTests(file: File): Option[List[FastTestRow]] = {
    if (file.exists) {
      val src = Source fromFile file
      val content = src.mkString
      src.close()

      Some(content.lines.toList map { line =>
        line.split(" ", 2) match {
          case Array(filename) => FastTestRow(filename)
          case Array(filename, args) => FastTestRow(filename, Some(args))
        }
      })
    } else {
      None
    }
  }

  private def saveToFile(file: File, rows: List[FastTestRow]): Unit = {
    val out = new PrintWriter(file)

    rows foreach { row =>
      out.println(row.toString())
    }

    out.flush()
    out.close()
  }

  private def fileForSuite(suite: Suite): File =
    new File(DefaultFastCacheName(suite.toString()))



  private def args(arguments: String): Option[String] =
    if (arguments == "") None else Some(arguments)

  private def rowFromTestable(tc: Testable): FastTestRow = {
    import tc.{ fileName, arguments }

    FastTestRow(fileName.toString(), args(arguments))
  }

  private def filterTestable(all: List[Testable with ExpectsOutput],
                             loaded: List[FastTestRow]): List[Testable with ExpectsOutput] = {
    val loadedSet = loaded.toSet

    all filter { tc => loadedSet contains rowFromTestable(tc) }
  }

  private def loadFastTests(suite: Suite): Option[List[Testable with ExpectsOutput]] = {
    loadFastTests(fileForSuite(suite)) map { fastTests =>
      suite match {
        case HipOnly()   => filterTestable(HipTestSuiteUsage.allTestable, fastTests)
        case SleekOnly() => filterTestable(SleekTestSuiteUsage.allTestable, fastTests)

        // This is a little more dubious,
        // since typically Sleek+Hip are treated distinctly.
        case All() => filterTestable(SleekTestSuiteUsage.allTestable ++ HipTestSuiteUsage.allTestable, fastTests)

        case SleekValidateOnly() => {
          // construct ValidateableSleekTestCase directly
          fastTests map { fastTest =>
            TestCaseBuilder(Paths.get("sleek"), Paths.get(fastTest.filename))
          }
        }
      }
    }
  }

  private def saveFastTests(suite: Suite, data: List[Testable]): Unit = {
    saveToFile(fileForSuite(suite), data map rowFromTestable)
  }

  private def allConstructTestCase(ps: PreparedSystem, tc: Testable with ExpectsOutput, conf: TestCaseConfiguration):
      TestCase = {
    if (tc.toString endsWith "hip") {
      HipTestCase.constructTestCase(ps, tc, conf)
    } else {
      // assume if-not-hip then must be sleek
      SleekTestCase.constructTestCase(ps, tc, conf)
    }
  }

  private def generateFastTestablesForSuite(suite: Suite): List[Testable] = {
    // get the universe of testable for the suite,
    val allTestable = suite match {
      case HipOnly()           => HipTestSuiteUsage.allTestable
      case SleekOnly()         => SleekTestSuiteUsage.allTestable
      case All()               => SleekTestSuiteUsage.allTestable ++ HipTestSuiteUsage.allTestable
      case SleekValidateOnly() => validate.allTestable
    }

    val construct: (PreparedSystem, Testable with ExpectsOutput, TestCaseConfiguration) => TestCase =
      suite match {
        case HipOnly()           => HipTestCase.constructTestCase
        case SleekOnly()         => SleekTestCase.constructTestCase

        // This is more involved;
        case All()               => allConstructTestCase
        case SleekValidateOnly() => ValidateableSleekTestCase.constructTestCase
      }

    // Try to keep the timing under 2 mins for running the tests
    val FastTestTime = 120 * 1000


    val resArch = new ResultsArchive(config.resultsDir, config.buildFailuresFile)
    val extantResults = allTestable map (resArch.resultsFor) filterNot (_.isEmpty)

    val extantTimeTestablePairs = extantResults map { res =>
      val timings = res map { case (rev, tcr) => tcr.executionTime }
      val avgTiming = timings.foldLeft(0L)({ (sum, time) => sum + time }) / timings.length

      val testable = res.head._2

      (avgTiming, testable)
    }
    val extantTime = extantTimeTestablePairs.foldLeft(0L) { (sum, pair) => sum + pair._1 }


    // if we have sufficent number of results, can compute from that,
    val timeTestablePairs = if (extantTime >= FastTestTime) {
      extantTimeTestablePairs
    } else {
      // otherwise, must run with a short timeout + don't save results (for T/O),
      // so as to see which tests are "quick"

      // Only save results for tests which take 10s or less.
      val QuickTimeout = 10
      val quickConfig = config.copy(saveResultOnTimeout = false,
                                    timeout = QuickTimeout)

      val runHipSleek = new RunHipSleek(quickConfig)
      import runHipSleek.{ altRunTests, runTestCaseForRevision }

      // If repo is dirty, this will be needlessly expensive.
      val repoC = repo.identify()

      // n.b. *might* throw UnableToBuildException here
      // XXX BUG: It seems this doesn't save the test case result, even for non-timeout?
      val tsr = altRunTests(construct, allTestable)(repoC)

      // wait for the results
      tsr.results map { tcr => (tcr.executionTime, tcr) }
    }


    val sortedTCRs = timeTestablePairs.sortBy { case (time, tc) => time }

    sortedTCRs.foldLeft(List[TestCaseResult]())({ (fastTests, timeTCPair) =>
      val (tcTime, tc) = timeTCPair

      val totalTime = fastTests.foldLeft(0L) { (sum, tcr) => sum + tcr.executionTime }

      if (totalTime < FastTestTime) {
        fastTests :+ tc
      } else {
        fastTests
      }
    })
  }

  private def suiteFromString(suite: String): Suite =
    suite match {
      case "hip" => HipOnly()
      case "sleek" => SleekOnly()
      case "all" => All()
      case "sleek-validate" => SleekValidateOnly()
      case "validate-sleek" => SleekValidateOnly()
      case _ => throw new IllegalArgumentException(s"Unknown suite: $suite")
    }

  def run(): Unit = {
{'fraction_non_alphanumeric': 0.09842243086159545, 'fraction_numerical': 0.002171552660152009, 'mean_word_length': 3.1533156498673742, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6201857', 'n_tokens_mistral': 4808, 'n_tokens_neox': 4646, 'n_words': 1189}