<commit_before>package dotty.tools.dotc
package sbt

import java.io.File
import java.util.{Arrays, EnumSet}

import dotty.tools.dotc.ast.Trees._
import dotty.tools.dotc.ast.tpd
import dotty.tools.dotc.core.Contexts._
import dotty.tools.dotc.core.Decorators._
import dotty.tools.dotc.core.Flags._
import dotty.tools.dotc.core.NameOps._
import dotty.tools.dotc.core.Names._
import dotty.tools.dotc.core.Phases._
import dotty.tools.dotc.core.StdNames._
import dotty.tools.dotc.core.Symbols._
import dotty.tools.dotc.core.Types._
import dotty.tools.dotc.transform.SymUtils._
import dotty.tools.io
import dotty.tools.io.{AbstractFile, PlainFile, ZipArchive}
import xsbti.UseScope
import xsbti.api.DependencyContext
import xsbti.api.DependencyContext._

import scala.collection.{Set, mutable}


/** This phase sends information on classes' dependencies to sbt via callbacks.
 *
 *  This is used by sbt for incremental recompilation. Briefly, when a file
 *  changes sbt will recompile it, if its API has changed (determined by what
 *  `ExtractAPI` sent) then sbt will determine which reverse-dependencies
 *  (determined by what `ExtractDependencies` sent) of the API have to be
 *  recompiled depending on what changed.
 *
 *  See the documentation of `ExtractDependenciesCollector`, `ExtractAPI`,
 *  `ExtractAPICollector` and
 *  http://www.scala-sbt.org/0.13/docs/Understanding-Recompilation.html for more
 *  information on how sbt incremental compilation works.
 *
 *  The following flags affect this phase:
 *   -Yforce-sbt-phases
 *   -Ydump-sbt-inc
 *
 *  @see ExtractAPI
 */
class ExtractDependencies extends Phase {
  import ExtractDependencies._

  override def phaseName: String = "sbt-deps"

  // This phase should be run directly after `Frontend`, if it is run after
  // `PostTyper`, some dependencies will be lost because trees get simplified.
  // See the scripted test `constants` for an example where this matters.
  // TODO: Add a `Phase#runsBefore` method ?

  override def run(implicit ctx: Context): Unit = {
    val unit = ctx.compilationUnit
    val dumpInc = ctx.settings.YdumpSbtInc.value
    val forceRun = dumpInc || ctx.settings.YforceSbtPhases.value
    val shouldRun = !unit.isJava && (ctx.sbtCallback != null || forceRun)

    if (shouldRun) {
      val collector = new ExtractDependenciesCollector
      collector.traverse(unit.tpdTree)

      if (dumpInc) {
        val deps = collector.dependencies.map(_.toString).toArray[Object]
        val names = collector.usedNames.map { case (clazz, names) => s"$clazz: $names" }.toArray[Object]
        Arrays.sort(deps)
        Arrays.sort(names)

        val pw = io.File(unit.source.file.jpath).changeExtension("inc").toFile.printWriter()
        // val pw = Console.out
        try {
          pw.println("Used Names:")
          pw.println("===========")
          names.foreach(pw.println)
          pw.println()
          pw.println("Dependencies:")
          pw.println("=============")
          deps.foreach(pw.println)
        } finally pw.close()
      }

      if (ctx.sbtCallback != null) {
        collector.usedNames.foreach {
          case (clazz, usedNames) =>
            val className = classNameAsString(clazz)
            usedNames.names.foreach {
              case (usedName, scopes) =>
                ctx.sbtCallback.usedName(className, usedName.toString, scopes)
            }
        }

        collector.dependencies.foreach(recordDependency)
      }
    }
  }

  /*
   * Handles dependency on given symbol by trying to figure out if represents a term
   * that is coming from either source code (not necessarily compiled in this compilation
   * run) or from class file and calls respective callback method.
   */
  def recordDependency(dep: ClassDependency)(implicit ctx: Context): Unit = {
    val fromClassName = classNameAsString(dep.from)
    val sourceFile = ctx.compilationUnit.source.file.file

    def binaryDependency(file: File, binaryClassName: String) =
      ctx.sbtCallback.binaryDependency(file, binaryClassName, fromClassName, sourceFile, dep.context)

    def processExternalDependency(depFile: AbstractFile) = {
      def binaryClassName(classSegments: List[String]) =
        classSegments.mkString(".").stripSuffix(".class")

      depFile match {
        case ze: ZipArchive#Entry => // The dependency comes from a JAR
          for (zip <- ze.underlyingSource; zipFile <- Option(zip.file)) {
            val classSegments = io.File(ze.path).segments
            binaryDependency(zipFile, binaryClassName(classSegments))
          }

        case pf: PlainFile => // The dependency comes from a class file
          val packages = dep.to.ownersIterator
            .count(x => x.is(PackageClass) && !x.isEffectiveRoot)
          // We can recover the fully qualified name of a classfile from
          // its path
          val classSegments = pf.givenPath.segments.takeRight(packages + 1)
          binaryDependency(pf.file, binaryClassName(classSegments))

        case _ =>
          ctx.warning(s"sbt-deps: Ignoring dependency $depFile of class ${depFile.getClass}}")
      }
    }

    val depFile = dep.to.associatedFile
    if (depFile != null) {<commit_msg>Fix crash when recording dependency information<commit_after>package dotty.tools.dotc
package sbt

import java.io.File
import java.util.{Arrays, EnumSet}

import dotty.tools.dotc.ast.Trees._
import dotty.tools.dotc.ast.tpd
import dotty.tools.dotc.core.Contexts._
import dotty.tools.dotc.core.Decorators._
import dotty.tools.dotc.core.Flags._
import dotty.tools.dotc.core.NameOps._
import dotty.tools.dotc.core.Names._
import dotty.tools.dotc.core.Phases._
import dotty.tools.dotc.core.StdNames._
import dotty.tools.dotc.core.Symbols._
import dotty.tools.dotc.core.Types._
import dotty.tools.dotc.transform.SymUtils._
import dotty.tools.io
import dotty.tools.io.{AbstractFile, PlainFile, ZipArchive}
import xsbti.UseScope
import xsbti.api.DependencyContext
import xsbti.api.DependencyContext._

import scala.collection.{Set, mutable}


/** This phase sends information on classes' dependencies to sbt via callbacks.
 *
 *  This is used by sbt for incremental recompilation. Briefly, when a file
 *  changes sbt will recompile it, if its API has changed (determined by what
 *  `ExtractAPI` sent) then sbt will determine which reverse-dependencies
 *  (determined by what `ExtractDependencies` sent) of the API have to be
 *  recompiled depending on what changed.
 *
 *  See the documentation of `ExtractDependenciesCollector`, `ExtractAPI`,
 *  `ExtractAPICollector` and
 *  http://www.scala-sbt.org/0.13/docs/Understanding-Recompilation.html for more
 *  information on how sbt incremental compilation works.
 *
 *  The following flags affect this phase:
 *   -Yforce-sbt-phases
 *   -Ydump-sbt-inc
 *
 *  @see ExtractAPI
 */
class ExtractDependencies extends Phase {
  import ExtractDependencies._

  override def phaseName: String = "sbt-deps"

  // This phase should be run directly after `Frontend`, if it is run after
  // `PostTyper`, some dependencies will be lost because trees get simplified.
  // See the scripted test `constants` for an example where this matters.
  // TODO: Add a `Phase#runsBefore` method ?

  override def run(implicit ctx: Context): Unit = {
    val unit = ctx.compilationUnit
    val dumpInc = ctx.settings.YdumpSbtInc.value
    val forceRun = dumpInc || ctx.settings.YforceSbtPhases.value
    val shouldRun = !unit.isJava && (ctx.sbtCallback != null || forceRun)

    if (shouldRun) {
      val collector = new ExtractDependenciesCollector
      collector.traverse(unit.tpdTree)

      if (dumpInc) {
        val deps = collector.dependencies.map(_.toString).toArray[Object]
        val names = collector.usedNames.map { case (clazz, names) => s"$clazz: $names" }.toArray[Object]
        Arrays.sort(deps)
        Arrays.sort(names)

        val pw = io.File(unit.source.file.jpath).changeExtension("inc").toFile.printWriter()
        // val pw = Console.out
        try {
          pw.println("Used Names:")
          pw.println("===========")
          names.foreach(pw.println)
          pw.println()
          pw.println("Dependencies:")
          pw.println("=============")
          deps.foreach(pw.println)
        } finally pw.close()
      }

      if (ctx.sbtCallback != null) {
        collector.usedNames.foreach {
          case (clazz, usedNames) =>
            val className = classNameAsString(clazz)
            usedNames.names.foreach {
              case (usedName, scopes) =>
                ctx.sbtCallback.usedName(className, usedName.toString, scopes)
            }
        }

        collector.dependencies.foreach(recordDependency)
      }
    }
  }

  /*
   * Handles dependency on given symbol by trying to figure out if represents a term
   * that is coming from either source code (not necessarily compiled in this compilation
   * run) or from class file and calls respective callback method.
   */
  def recordDependency(dep: ClassDependency)(implicit ctx: Context): Unit = {
    val fromClassName = classNameAsString(dep.from)
    val sourceFile = ctx.compilationUnit.source.file.file

    def binaryDependency(file: File, binaryClassName: String) =
      ctx.sbtCallback.binaryDependency(file, binaryClassName, fromClassName, sourceFile, dep.context)

    def processExternalDependency(depFile: AbstractFile) = {
      def binaryClassName(classSegments: List[String]) =
        classSegments.mkString(".").stripSuffix(".class")

      depFile match {
        case ze: ZipArchive#Entry => // The dependency comes from a JAR
          for (zip <- ze.underlyingSource; zipFile <- Option(zip.file)) {
            val classSegments = io.File(ze.path).segments
            binaryDependency(zipFile, binaryClassName(classSegments))
          }

        case pf: PlainFile => // The dependency comes from a class file
          val packages = dep.to.ownersIterator
            .count(x => x.is(PackageClass) && !x.isEffectiveRoot)
          // We can recover the fully qualified name of a classfile from
          // its path
          val classSegments = pf.givenPath.segments.takeRight(packages + 1)
          // FIXME: pf.file is null for classfiles coming from the modulepath
          // (handled by JrtClassPath) because they cannot be represented as
          // java.io.File, since the `binaryDependency` callback must take a
          // java.io.File, this means that we cannot record dependencies coming
          // from the modulepath. For now this isn't a big deal since we only
          // support having the standard Java library on the modulepath.
          if (pf.file != null)
            binaryDependency(pf.file, binaryClassName(classSegments))

        case _ =>
          ctx.warning(s"sbt-deps: Ignoring dependency $depFile of class ${depFile.getClass}}")
      }
    }

    val depFile = dep.to.associatedFile
    if (depFile != null) {
{'fraction_non_alphanumeric': 0.10263929618768329, 'fraction_numerical': 0.0007331378299120235, 'mean_word_length': 2.985756026296567, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4245528', 'n_tokens_mistral': 3255, 'n_tokens_neox': 3130, 'n_words': 872}