<commit_before>package dotty.tools.vulpix

import scala.io.Source
import scala.util.Using
import java.io.File
import java.lang.System.{lineSeparator => EOL}
import java.nio.file.{Files, Paths}

object FileDiff {
  def diffMessage(expectFile: String, actualFile: String): String =
      s"""Test output dumped in: $actualFile
          |  See diff of the checkfile (`brew install icdiff` for colored diff)
          |    > diff $expectFile $actualFile
          |  Replace checkfile with current output
          |    > mv $actualFile $expectFile
      """.stripMargin

  def check(sourceTitle: String, outputLines: Seq[String], checkFile: String): Option[String] = {
    val checkLines =
      if (!(new File(checkFile)).exists) Nil
      else Using(Source.fromFile(checkFile, "UTF-8"))(_.getLines().toList).get

    // handle check file path mismatch on windows
    def lineMatch(out: String, expect: String): Boolean =
      out == expect || File.separatorChar == '\\' && out.replace('\\', '/') == expect

    def linesMatch =
      outputLines.length == checkLines.length &&
      outputLines.lazyZip(checkLines).forall(lineMatch)

    if (!linesMatch) Some(
      s"""|Output from '$sourceTitle' did not match check file. Actual output:
          |${outputLines.mkString(EOL)}
          |""".stripMargin + "\n")
    else None
  }

  def dump(path: String, content: Seq[String]): Unit = {
    val outFile = dotty.tools.io.File(path)
    outFile.writeAll(content.mkString("", EOL, EOL))
  }

  def checkAndDump(sourceTitle: String, actualLines: Seq[String], checkFilePath: String): Boolean = {
    val outFilePath = checkFilePath + ".out"
    FileDiff.check(sourceTitle, actualLines, checkFilePath) match {
      case Some(msg) =>
        FileDiff.dump(outFilePath, actualLines)<commit_msg>Ignore windows line endings in line comparison<commit_after>package dotty.tools.vulpix

import scala.io.Source
import scala.util.Using
import java.io.File
import java.lang.System.{lineSeparator => EOL}
import java.nio.file.{Files, Paths}

object FileDiff {
  def diffMessage(expectFile: String, actualFile: String): String =
      s"""Test output dumped in: $actualFile
          |  See diff of the checkfile (`brew install icdiff` for colored diff)
          |    > diff $expectFile $actualFile
          |  Replace checkfile with current output
          |    > mv $actualFile $expectFile
      """.stripMargin

  def check(sourceTitle: String, outputLines: Seq[String], checkFile: String): Option[String] = {
    val checkLines =
      if (!(new File(checkFile)).exists) Nil
      else Using(Source.fromFile(checkFile, "UTF-8"))(_.getLines().toList).get

    // handle check file path mismatch on windows
    def lineMatch(actual: String, expect: String): Boolean = {
      val actual1 = actual.stripLineEnd
      val expect1  = expect.stripLineEnd
      actual1 == expect1 || File.separatorChar == '\\' && actual1.replace('\\', '/') == expect1
    }

    def linesMatch =
      outputLines.length == checkLines.length &&
      outputLines.lazyZip(checkLines).forall(lineMatch)

    if (!linesMatch) Some(
      s"""|Output from '$sourceTitle' did not match check file. Actual output:
          |${outputLines.mkString(EOL)}
          |""".stripMargin + "\n")
    else None
  }

  def dump(path: String, content: Seq[String]): Unit = {
    val outFile = dotty.tools.io.File(path)
    outFile.writeAll(content.mkString("", EOL, EOL))
  }

  def checkAndDump(sourceTitle: String, actualLines: Seq[String], checkFilePath: String): Boolean = {
    val outFilePath = checkFilePath + ".out"
    FileDiff.check(sourceTitle, actualLines, checkFilePath) match {
      case Some(msg) =>
        FileDiff.dump(outFilePath, actualLines)
{'fraction_non_alphanumeric': 0.1258402796450659, 'fraction_numerical': 0.0021511158913686476, 'mean_word_length': 3.335664335664336, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '521125', 'n_tokens_mistral': 1123, 'n_tokens_neox': 1098, 'n_words': 291}