<commit_before>
import java.nio.file.{Files, Path, Paths}
import java.util.stream.{ Stream => JStream }

import scala.collection.JavaConverters._

object IdempotencyCheck {
  val blacklistedSet = Set(
    // No fix needed. Bridges on collections in different order. Second one in scala2 order.
    "pos/Map/scala/collection/immutable/Map",
    "pos/Map/scala/collection/immutable/AbstractMap",
    "pos/t1203a/NodeSeq",
    "pos/i2345/Whatever"
  )

  def blacklisted(s: String) = {
    blacklistedSet(s) ||
    // FIXME: non-deterministic method/class names
    s.contains("$$anon$") ||
    s.contains("$$anonfun$") ||
    s.contains("$accu$") ||
    s.contains("$accum$") ||
    s.startsWith("/dotty/dotty/tools/dotc/sbt/ExtractAPICollector$classFirstSort$") ||
    s.startsWith("/dotty/dotty/tools/dotc/transform/PatternMatcher$Translator$TreeMakers$IntEqualityTestTreeMaker$") ||
    s.startsWith("dotty/dotty/tools/dotc/transform/Constructors$intoConstr$") ||
    s.startsWith("/dotty/dotty/tools/dotc/typer/RefChecks$MixinOverrideError$") ||
    s.startsWith("/dotty/dotty/tools/dotc/typer/ImplicitRunInfo$liftToClasses$") ||
    s.startsWith("/dotty/dotty/tools/dotc/typer/Inliner$addAccessors$") ||
    s.startsWith("/dotty/dotty/tools/dotc/typer/ErrorReporting$reported$")
  }

  def checkIdempotency(dirPrefix: String): Unit = {
    var failed = 0
    var total = 0

    val groupedBytecodeFiles: List[(Path, Path, Path, Path)] = {
      val bytecodeFiles = {
        def bytecodeFiles(paths: JStream[Path]): List[Path] = {
          def isBytecode(file: String) = file.endsWith(".class") || file.endsWith(".tasty")
          paths.iterator.asScala.filter(path => isBytecode(path.toString)).toList
        }
        val compilerDir1 = Paths.get(dirPrefix + 1)
        val compilerDir2 = Paths.get(dirPrefix + 2)
        bytecodeFiles(Files.walk(compilerDir1)) ++ bytecodeFiles(Files.walk(compilerDir2))
      }
      val groups = bytecodeFiles.groupBy(f => f.toString.substring(dirPrefix.length + 1, f.toString.length - 6))

      groups.filterNot(x => blacklisted(x._1)).valuesIterator.flatMap { g =><commit_msg>Remove blacklisted inner method and objects<commit_after>
import java.nio.file.{Files, Path, Paths}
import java.util.stream.{ Stream => JStream }

import scala.collection.JavaConverters._

object IdempotencyCheck {
  val blacklisted = Set(
    // No fix needed. Bridges on collections in different order. Second one in scala2 order.
    "pos/Map/scala/collection/immutable/Map",
    "pos/Map/scala/collection/immutable/AbstractMap",
    "pos/t1203a/NodeSeq",
    "pos/i2345/Whatever"
  )

  def checkIdempotency(dirPrefix: String): Unit = {
    var failed = 0
    var total = 0

    val groupedBytecodeFiles: List[(Path, Path, Path, Path)] = {
      val bytecodeFiles = {
        def bytecodeFiles(paths: JStream[Path]): List[Path] = {
          def isBytecode(file: String) = file.endsWith(".class") || file.endsWith(".tasty")
          paths.iterator.asScala.filter(path => isBytecode(path.toString)).toList
        }
        val compilerDir1 = Paths.get(dirPrefix + 1)
        val compilerDir2 = Paths.get(dirPrefix + 2)
        bytecodeFiles(Files.walk(compilerDir1)) ++ bytecodeFiles(Files.walk(compilerDir2))
      }
      val groups = bytecodeFiles.groupBy(f => f.toString.substring(dirPrefix.length + 1, f.toString.length - 6))

      groups.filterNot(x => blacklisted(x._1)).valuesIterator.flatMap { g =>
{'fraction_non_alphanumeric': 0.14231664726426077, 'fraction_numerical': 0.011641443538998836, 'mean_word_length': 3.638326585695007, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '514055', 'n_tokens_mistral': 1170, 'n_tokens_neox': 1142, 'n_words': 190}