<commit_before>package edu.nus.systemtesting.testsuite

import edu.nus.systemtesting.TestCaseResult
import edu.nus.systemtesting.output.GlobalReporter

import GlobalReporter.reporter

/**
 * For comparing two [[TestSuiteResult]]s. Does not assume that the two
 * results were computed from the same list of [[TestCase]]s, since
 * the list of tests may change over time.
 *
 * Result of the comparison indicates the differences in results of the
 * [[TestSuite]] executions, and notes incompatibilities.
 *
 * Use [[TestSuiteComparison.apply]] to compute the 'diff' of two
 * [[TestSuiteResult]]s.
 *
 * @author richardg
 */
class TestSuiteComparison(val oldRevision: String,
                          val curRevision: String,
                          val argsChangedTests: List[(TestCaseResult, TestCaseResult)],
                          val removedTests: List[TestCaseResult],
                          val newTests: List[TestCaseResult],
                          val nowSuccessfullyRuns: List[(TestCaseResult, TestCaseResult)],
                          val usedToSuccessfullyRun: List[(TestCaseResult, TestCaseResult)],
                          val nowPasses: List[(TestCaseResult, TestCaseResult)],
                          val usedToPass: List[(TestCaseResult, TestCaseResult)],
                          val diffDiffs: List[(TestCaseResult, TestCaseResult)]) {
  val unchanged = {
    List(argsChangedTests,
         removedTests,
         newTests,
         nowSuccessfullyRuns,
         usedToSuccessfullyRun,
         nowPasses,
         usedToPass,
         diffDiffs)
      .forall(_.isEmpty)
  }

  def displayResult(): Unit = {
    reporter.header(s"Diff between results $oldRevision -> $curRevision")

    if (unchanged) {
      reporter.log("No differences.")
      return
    }

    // Convenient way to display TestCaseResult
    def tcrToString(tcr: TestCaseResult): String = {
      import tcr.{ command, arguments, filename }
      s"TC[$command, $arguments, $filename]"
    }

    if (!argsChangedTests.isEmpty) {
      // TODO: at the moment, test cases haven't changed, so we don't need this.
      println("argsChanged, but this shouldn't be the case")
    }

    if (!removedTests.isEmpty) {
      // TODO: at the moment, test cases haven't changed, so we don't need this.
      println("removedTests, but this shouldn't be the case")
    }

    if (!newTests.isEmpty) {
      // TODO: at the moment, test cases haven't changed, so we don't need this.
      println("newTests, but this shouldn't be the case")
      newTests.foreach({ tc =>
        reporter.log(s"* ${tcrToString(tc)}")
      })
    }

    def reportListOfPairs(ls: List[(TestCaseResult, TestCaseResult)], title: String, sym: String = "*"): Unit = {
      if (!ls.isEmpty) {
        reporter.log(s"$title:")
        ls.foreach({ case (_, tc) =>
          // Just show the new TC, not the old one.
          reporter.log(s"$sym ${tcrToString(tc)}")
        })
        reporter.println()
      }
    }

    reportListOfPairs(nowSuccessfullyRuns, "Now runs successfully", "+")
    reportListOfPairs(usedToSuccessfullyRun, "Used to run successfully", "-")

    reportListOfPairs(nowPasses, "Now passes", "+")
    reportListOfPairs(usedToPass, "Used to pass", "-")

    // Might be useful to show the diffs.
    // Might not be best to show this one last, also.
    reportListOfPairs(diffDiffs, "Both fail, but different diffs", "*")
  }
}

object TestSuiteComparison {
  def apply(oldTS: TestSuiteResult, curTS: TestSuiteResult): TestSuiteComparison = {
    // It can't be assumed that oldTS, currTS ran the same set of TestCases
    // (in terms of *SuiteUsage), since these can change over time as TestCases
    // are added, program flags change, etc.
    //
    // Still, roughly assumed that the two SuiteResults are running *roughly*
    // the same set of tests.
    // Somewhat assumes corpusDir is the same, but in theory it could be different
    // (but with same set of files) and it'd work just fine, too.

    val oldRevision = oldTS.repoRevision
    val curRevision = curTS.repoRevision

    //
    // Find which TestCaseResults can be compared, which can't
    //

    // Build Map[(cmd,filename,sortedArgs) => TestCaseResult)] for old,curr
    val oldTests = oldTS.results.toList
    val curTests = curTS.results.toList

    val oldTestMap = oldTests.map(tcr => (tcr.cmdFnArgsKey -> tcr)).toMap
    val curTestMap = curTests.map(tcr => (tcr.cmdFnArgsKey -> tcr)).toMap

    // Partition (Compatible TestCaseResults)<commit_msg>More detailed output from TestSuite Comparison<commit_after>package edu.nus.systemtesting.testsuite

import edu.nus.systemtesting.TestCaseResult
import edu.nus.systemtesting.output.GlobalReporter

import GlobalReporter.reporter

/**
 * For comparing two [[TestSuiteResult]]s. Does not assume that the two
 * results were computed from the same list of [[TestCase]]s, since
 * the list of tests may change over time.
 *
 * Result of the comparison indicates the differences in results of the
 * [[TestSuite]] executions, and notes incompatibilities.
 *
 * Use [[TestSuiteComparison.apply]] to compute the 'diff' of two
 * [[TestSuiteResult]]s.
 *
 * @author richardg
 */
class TestSuiteComparison(val oldRevision: String,
                          val curRevision: String,
                          val argsChangedTests: List[(TestCaseResult, TestCaseResult)],
                          val removedTests: List[TestCaseResult],
                          val newTests: List[TestCaseResult],
                          val nowSuccessfullyRuns: List[(TestCaseResult, TestCaseResult)],
                          val usedToSuccessfullyRun: List[(TestCaseResult, TestCaseResult)],
                          val nowPasses: List[(TestCaseResult, TestCaseResult)],
                          val usedToPass: List[(TestCaseResult, TestCaseResult)],
                          val diffDiffs: List[(TestCaseResult, TestCaseResult)]) {
  val unchanged = {
    List(argsChangedTests,
         removedTests,
         newTests,
         nowSuccessfullyRuns,
         usedToSuccessfullyRun,
         nowPasses,
         usedToPass,
         diffDiffs)
      .forall(_.isEmpty)
  }

  def displayResult(): Unit = {
    reporter.header(s"Diff between results $oldRevision -> $curRevision")

    if (unchanged) {
      reporter.log("No differences.")
      return
    }

    // Convenient way to display TestCaseResult
    def tcrToString(tcr: TestCaseResult): String = {
      import tcr.{ command, arguments, filename }
      s"TC[$command, $arguments, $filename]"
    }

    if (!argsChangedTests.isEmpty) {
      reporter.log("Args changed:")
      argsChangedTests foreach { case (oldTCR, curTCR) =>
        reporter.log(s"${tcrToString(curTCR)}: args were `${oldTCR.arguments}`")
      }
      reporter.println()
    }

    if (!removedTests.isEmpty) {
      reporter.log("Tests Removed:")
      removedTests foreach { rmTCR =>
        reporter.log(s"- ${tcrToString(rmTCR)}")
      }
      reporter.println()
    }

    if (!newTests.isEmpty) {
      reporter.log("Tests Added:")
      newTests.foreach({ tc =>
        reporter.log(s"+ ${tcrToString(tc)}")
      })
      reporter.println()
    }

    def reportListOfPairs(ls: List[(TestCaseResult, TestCaseResult)], title: String, sym: String = "*"): Unit = {
      if (!ls.isEmpty) {
        reporter.log(s"$title:")
        ls.foreach({ case (_, tc) =>
          // Just show the new TC, not the old one.
          reporter.log(s"$sym ${tcrToString(tc)}")
        })
        reporter.println()
      }
    }

    reportListOfPairs(nowSuccessfullyRuns, "Now runs successfully", "+")
    reportListOfPairs(usedToSuccessfullyRun, "Used to run successfully", "-")

    reportListOfPairs(nowPasses, "Now passes", "+")

    reportListOfPairs(usedToPass, "Used to pass", "-")

    // Might be useful to show the diffs.
    // Might not be best to show this one last, also.
    reportListOfPairs(diffDiffs, "Both fail, but different diffs", "*")
    if (!diffDiffs.isEmpty) {
      reporter.log(s"Fails, but different diffs:")
      diffDiffs.foreach({ case (oldTCR, curTCR) =>
        val oldDiff = oldTCR diff
        val curDiff = curTCR diff

        reporter log s"* ${tcrToString(curTCR)}:"
        reporter log s"Was:"
        oldDiff foreach { res => reporter println res.toString() }
        reporter log s"Now:"
        curDiff foreach { res => reporter println res.toString() }
      })
      reporter.println()
    }
  }
}

object TestSuiteComparison {
  def apply(oldTS: TestSuiteResult, curTS: TestSuiteResult): TestSuiteComparison = {
    // It can't be assumed that oldTS, currTS ran the same set of TestCases
    // (in terms of *SuiteUsage), since these can change over time as TestCases
    // are added, program flags change, etc.
    //
    // Still, roughly assumed that the two SuiteResults are running *roughly*
    // the same set of tests.
    // Somewhat assumes corpusDir is the same, but in theory it could be different
    // (but with same set of files) and it'd work just fine, too.

    val oldRevision = oldTS.repoRevision
    val curRevision = curTS.repoRevision

    //
    // Find which TestCaseResults can be compared, which can't
    //

    // Build Map[(cmd,filename,sortedArgs) => TestCaseResult)] for old,curr
    val oldTests = oldTS.results.toList
    val curTests = curTS.results.toList

    val oldTestMap = oldTests.map(tcr => (tcr.cmdFnArgsKey -> tcr)).toMap
    val curTestMap = curTests.map(tcr => (tcr.cmdFnArgsKey -> tcr)).toMap

    // Partition (Compatible TestCaseResults)
{'fraction_non_alphanumeric': 0.10746299770690014, 'fraction_numerical': 0.0, 'mean_word_length': 2.6594202898550723, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1892328', 'n_tokens_mistral': 2769, 'n_tokens_neox': 2709, 'n_words': 796}