<commit_before>package org.jetbrains.plugins.scala
package lang
package parser

import com.intellij.psi.PsiErrorElement
import com.intellij.psi.impl.DebugUtil.psiToString
import org.jetbrains.plugins.scala.base.SimpleTestCase
import org.jetbrains.plugins.scala.extensions._
import org.jetbrains.plugins.scala.lang.psi.api.ScalaFile
import org.junit.Assert.assertEquals

trait SimpleScala3ParserTestBase extends SimpleTestCase {
  def err(err: String): String = {
    s"[[Err($err)]]"
  }
  private val errRegex = raw"\[\[Err\((.+)\)\]\]".r

  private def extractExpectedErrors(text: String): (String, Set[(Int, String)]) = {
    var code = text
    val errorsBuilder = Set.newBuilder[(Int, String)]
    while(true) {
      errRegex.findFirstMatchIn(code) match {
        case Some(matc) =>
          code = code.substring(0, matc.start) + code.substring(matc.end)
          errorsBuilder += (matc.start -> matc.group(1))
        case None =>
          return code -> errorsBuilder.result()
      }
    }
    ???
  }

  def checkParseErrors(text: String): ScalaFile = {
    val (code, expectedErrors) = extractExpectedErrors(text)

    val file = parseText(code, lang = Scala3Language.INSTANCE)
    val errors = file.depthFirst().toSeq.filterBy[PsiErrorElement]
    if (expectedErrors.isEmpty) {
      assert(errors.isEmpty, "Expected no errors but found: " + errors.map(_.getErrorDescription).mkString(", "))
    } else {
      val errorsWithPos = errors.map(psi => psi.getTextOffset -> psi.getErrorDescription).toSet
      val notFoundErrors = (expectedErrors -- errorsWithPos).toSeq.sortBy(_._1).map(_._2)
      val unexpectedErrors = (errorsWithPos -- expectedErrors).toSeq.sortBy(_._1).map(_._2)
      assert(unexpectedErrors.isEmpty, "Found unexpected errors: " + unexpectedErrors.mkString(", "))
      assert(notFoundErrors.isEmpty, "Expected errors but didn't find: " + notFoundErrors.mkString(", "))
    }
    file
  }

  def checkTree(code: String, expectedTree: String): Unit = {
    val file = parseText(code, lang = Scala3Language.INSTANCE)
    val resultTree = psiToString(file, false).replace(": " + file.name, "")
    assertEquals(expectedTree.trim, resultTree.trim)
  }
}<commit_msg>fixed new line problem in dotty parser test<commit_after>package org.jetbrains.plugins.scala
package lang
package parser

import com.intellij.psi.PsiErrorElement
import com.intellij.psi.impl.DebugUtil.psiToString
import org.jetbrains.plugins.scala.base.SimpleTestCase
import org.jetbrains.plugins.scala.extensions._
import org.jetbrains.plugins.scala.lang.psi.api.ScalaFile
import org.junit.Assert.assertEquals

trait SimpleScala3ParserTestBase extends SimpleTestCase {
  def err(err: String): String = {
    s"[[Err($err)]]"
  }
  private val errRegex = raw"\[\[Err\((.+)\)\]\]".r

  private def extractExpectedErrors(text: String): (String, Set[(Int, String)]) = {
    var code = text
    val errorsBuilder = Set.newBuilder[(Int, String)]
    while(true) {
      errRegex.findFirstMatchIn(code) match {
        case Some(matc) =>
          code = code.substring(0, matc.start) + code.substring(matc.end)
          errorsBuilder += (matc.start -> matc.group(1))
        case None =>
          return code -> errorsBuilder.result()
      }
    }
    ???
  }

  def checkParseErrors(text: String): ScalaFile = {
    val (code, expectedErrors) = extractExpectedErrors(text)

    val file = parseText(code, lang = Scala3Language.INSTANCE)
    val errors = file.depthFirst().toSeq.filterBy[PsiErrorElement]
    if (expectedErrors.isEmpty) {
      assert(errors.isEmpty, "Expected no errors but found: " + errors.map(_.getErrorDescription).mkString(", "))
    } else {
      val errorsWithPos = errors.map(psi => psi.getTextOffset -> psi.getErrorDescription).toSet
      val notFoundErrors = (expectedErrors -- errorsWithPos).toSeq.sortBy(_._1).map(_._2)
      val unexpectedErrors = (errorsWithPos -- expectedErrors).toSeq.sortBy(_._1).map(_._2)
      assert(unexpectedErrors.isEmpty, "Found unexpected errors: " + unexpectedErrors.mkString(", "))
      assert(notFoundErrors.isEmpty, "Expected errors but didn't find: " + notFoundErrors.mkString(", "))
    }
    file
  }

  def checkTree(code: String, expectedTree: String): Unit = {
    val file = parseText(code.withNormalizedSeparator, lang = Scala3Language.INSTANCE)
    val resultTree = psiToString(file, false).replace(": " + file.name, "")
    assertEquals(expectedTree.trim.withNormalizedSeparator, resultTree.trim)
  }
}
{'fraction_non_alphanumeric': 0.13181411974977658, 'fraction_numerical': 0.004021447721179625, 'mean_word_length': 3.824353448275862, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2149117', 'n_tokens_mistral': 1373, 'n_tokens_neox': 1351, 'n_words': 271}