<commit_before>package org.scalaide.core.internal.decorators.semantichighlighting.classifier

import scala.PartialFunction.condOpt
import scala.collection.mutable
import org.scalaide.core.compiler.ScalaPresentationCompiler
import org.scalaide.logging.HasLogger
import org.scalaide.core.internal.decorators.semantichighlighting.classifier.SymbolTypes._
import scala.tools.nsc.io.AbstractFile
import scala.tools.nsc.util.RangePosition
import scala.reflect.internal.util.SourceFile
import org.eclipse.core.runtime.IProgressMonitor
import org.eclipse.jface.text.IRegion
import org.eclipse.jface.text.Region
import org.scalaide.util.internal.eclipse.RegionOps._

private object SymbolClassification {

  /**
   *  If a symbol gets classified as more than one type, we give certain types precedence.
   *  Preference is given to map values over the corresponding key.
   */
  private val pruneTable: Map[SymbolType, Set[SymbolType]] = Map(
    LocalVar -> Set(LazyLocalVal),
    LocalVal -> Set(LocalVar, TemplateVal, TemplateVar, LazyTemplateVal, LazyLocalVal),
    Method -> Set(TemplateVal, TemplateVar, LazyTemplateVal),
    Class -> Set(Type, CaseClass, Object),
    CaseClass -> Set(CaseObject),
    Param -> Set(TemplateVal, TemplateVar, LazyTemplateVal),
    TemplateVal -> Set(Type),
    Object -> Set(CaseClass)
  )

}

class SymbolClassification(protected val sourceFile: SourceFile, val global: ScalaPresentationCompiler, useSyntacticHints: Boolean)
  extends SafeSymbol with TypeTreeTraverser with SymbolTests with HasLogger {

  import SymbolClassification._
  import global.Symbol
  import global.Position
  import global.NoSymbol

  def compilationUnitOfFile(f: AbstractFile) = global.unitOfFile.get(f)

  protected lazy val syntacticInfo =
    if (useSyntacticHints) SyntacticInfo.getSyntacticInfo(sourceFile.content.mkString) else SyntacticInfo.noSyntacticInfo

  private lazy val unitTree: global.Tree = global.loadedType(sourceFile, true).fold(identity, _ => global.EmptyTree)

  private def canSymbolBeReferencedInSource(sym: Symbol): Boolean = {
    def isSyntheticMethodParam(sym: Symbol): Boolean = sym.isSynthetic && sym.isValueParameter

    !sym.isAnonymousFunction &&
    !sym.isAnonymousClass &&
    !isSyntheticMethodParam(sym)
  }

  def classifySymbols(progressMonitor: IProgressMonitor): List[SymbolInfo] = {
    val rawSymbolInfos: Seq[SymbolInfo] = {
      import global._

      def getSymbolInfo(symbolType: SymbolType, sym: Symbol, region: Option[IRegion]): SymbolInfo = {
        val inInterpolatedString = region.map(syntacticInfo.identifiersInStringInterpolations).getOrElse(false)
        // isDeprecated may trigger type completion for annotations
        val deprecated = sym.annotations.nonEmpty && global.askOption(() => sym.isDeprecated).getOrElse(false)
        SymbolInfo(symbolType, region.toList, deprecated, inInterpolatedString)
      }

      def findDynamicInfo(t: Tree): Option[SymbolInfo] = {
        def posToRegion(p: Position) =
          new Region(p.start, p.end - p.start)

        findDynamicMethodCall(t) map {
          case (symbolType, pos) =>
            val sym = global.askOption(() => t.symbol).getOrElse(NoSymbol)
            getSymbolInfo(symbolType, sym, Some(posToRegion(pos)))
        }
      }

      def findSymbolInfo(t: Tree): List[SymbolInfo] =
        safeSymbol(t) collect {
          case (sym, pos) if canSymbolBeReferencedInSource(sym) =>
            getSymbolInfo(getSymbolType(sym), sym, getOccurrenceRegion(sym)(pos))
        }

      var symbolInfos = IndexedSeq.empty[SymbolInfo]
      new Traverser {
        override def traverse(t: Tree): Unit = {
          if (!progressMonitor.isCanceled() && (t.symbol != NoSymbol || t.isType) && isSourceTree(t)) {
            val ds = findDynamicInfo(t)
            val xs = if (ds.isEmpty) findSymbolInfo(t) else ds.toList
            symbolInfos ++= xs
          }
          super.traverse(t)
        }
      }.traverse(unitTree)
      symbolInfos
    }
    if (progressMonitor.isCanceled()) return Nil

    val prunedSymbolInfos = prune(rawSymbolInfos)
    if (progressMonitor.isCanceled()) return Nil

    val all: Set[IRegion] = rawSymbolInfos.flatMap(_.regions).toSet
    if (progressMonitor.isCanceled()) return Nil

    val localVars: Set[IRegion] = rawSymbolInfos.collect { case SymbolInfo(LocalVar, regions, _, _) => regions }.flatten.toSet
    if (progressMonitor.isCanceled()) return Nil

    val symbolInfosFromSyntax = getSymbolInfosFromSyntax(syntacticInfo, localVars, all)
    if (progressMonitor.isCanceled()) return Nil

    (symbolInfosFromSyntax ++ prunedSymbolInfos).filter(_.regions.nonEmpty).distinct
  }

  private def getOccurrenceRegion(sym: Symbol)(pos: Position): Option[IRegion] =
    getNameRegion(pos) flatMap { region =>
      val text = region of sourceFile.content<commit_msg>Fix race condition due to call of `Tree.symbol`<commit_after>package org.scalaide.core.internal.decorators.semantichighlighting.classifier

import scala.PartialFunction.condOpt
import scala.collection.mutable
import org.scalaide.core.compiler.ScalaPresentationCompiler
import org.scalaide.logging.HasLogger
import org.scalaide.core.internal.decorators.semantichighlighting.classifier.SymbolTypes._
import scala.tools.nsc.io.AbstractFile
import scala.tools.nsc.util.RangePosition
import scala.reflect.internal.util.SourceFile
import org.eclipse.core.runtime.IProgressMonitor
import org.eclipse.jface.text.IRegion
import org.eclipse.jface.text.Region
import org.scalaide.util.internal.eclipse.RegionOps._

private object SymbolClassification {

  /**
   *  If a symbol gets classified as more than one type, we give certain types precedence.
   *  Preference is given to map values over the corresponding key.
   */
  private val pruneTable: Map[SymbolType, Set[SymbolType]] = Map(
    LocalVar -> Set(LazyLocalVal),
    LocalVal -> Set(LocalVar, TemplateVal, TemplateVar, LazyTemplateVal, LazyLocalVal),
    Method -> Set(TemplateVal, TemplateVar, LazyTemplateVal),
    Class -> Set(Type, CaseClass, Object),
    CaseClass -> Set(CaseObject),
    Param -> Set(TemplateVal, TemplateVar, LazyTemplateVal),
    TemplateVal -> Set(Type),
    Object -> Set(CaseClass)
  )

}

class SymbolClassification(protected val sourceFile: SourceFile, val global: ScalaPresentationCompiler, useSyntacticHints: Boolean)
  extends SafeSymbol with TypeTreeTraverser with SymbolTests with HasLogger {

  import SymbolClassification._
  import global.Symbol
  import global.Position
  import global.NoSymbol

  def compilationUnitOfFile(f: AbstractFile) = global.unitOfFile.get(f)

  protected lazy val syntacticInfo =
    if (useSyntacticHints) SyntacticInfo.getSyntacticInfo(sourceFile.content.mkString) else SyntacticInfo.noSyntacticInfo

  private lazy val unitTree: global.Tree = global.loadedType(sourceFile, true).fold(identity, _ => global.EmptyTree)

  private def canSymbolBeReferencedInSource(sym: Symbol): Boolean = {
    def isSyntheticMethodParam(sym: Symbol): Boolean = sym.isSynthetic && sym.isValueParameter

    !sym.isAnonymousFunction &&
    !sym.isAnonymousClass &&
    !isSyntheticMethodParam(sym)
  }

  def classifySymbols(progressMonitor: IProgressMonitor): List[SymbolInfo] = {
    val rawSymbolInfos: Seq[SymbolInfo] = {
      import global._

      def getSymbolInfo(symbolType: SymbolType, sym: Symbol, region: Option[IRegion]): SymbolInfo = {
        val inInterpolatedString = region.map(syntacticInfo.identifiersInStringInterpolations).getOrElse(false)
        // isDeprecated may trigger type completion for annotations
        val deprecated = sym.annotations.nonEmpty && global.askOption(() => sym.isDeprecated).getOrElse(false)
        SymbolInfo(symbolType, region.toList, deprecated, inInterpolatedString)
      }

      def findDynamicInfo(t: Tree): Option[SymbolInfo] = {
        def posToRegion(p: Position) =
          new Region(p.start, p.end - p.start)

        findDynamicMethodCall(t) map {
          case (symbolType, pos) =>
            val sym = global.askOption(() => t.symbol).getOrElse(NoSymbol)
            getSymbolInfo(symbolType, sym, Some(posToRegion(pos)))
        }
      }

      def findSymbolInfo(t: Tree): List[SymbolInfo] =
        safeSymbol(t) collect {
          case (sym, pos) if canSymbolBeReferencedInSource(sym) =>
            getSymbolInfo(getSymbolType(sym), sym, getOccurrenceRegion(sym)(pos))
        }

      var symbolInfos = IndexedSeq.empty[SymbolInfo]
      new Traverser {
        override def traverse(t: Tree): Unit = {
          def symExists = global.askOption(() => t.symbol != NoSymbol).getOrElse(false)

          if (!progressMonitor.isCanceled() && isSourceTree(t) && (t.hasSymbol || t.isType || symExists)) {
            val ds = findDynamicInfo(t)
            val xs = if (ds.isEmpty) findSymbolInfo(t) else ds.toList
            symbolInfos ++= xs
          }
          super.traverse(t)
        }
      }.traverse(unitTree)
      symbolInfos
    }
    if (progressMonitor.isCanceled()) return Nil

    val prunedSymbolInfos = prune(rawSymbolInfos)
    if (progressMonitor.isCanceled()) return Nil

    val all: Set[IRegion] = rawSymbolInfos.flatMap(_.regions).toSet
    if (progressMonitor.isCanceled()) return Nil

    val localVars: Set[IRegion] = rawSymbolInfos.collect { case SymbolInfo(LocalVar, regions, _, _) => regions }.flatten.toSet
    if (progressMonitor.isCanceled()) return Nil

    val symbolInfosFromSyntax = getSymbolInfosFromSyntax(syntacticInfo, localVars, all)
    if (progressMonitor.isCanceled()) return Nil

    (symbolInfosFromSyntax ++ prunedSymbolInfos).filter(_.regions.nonEmpty).distinct
  }

  private def getOccurrenceRegion(sym: Symbol)(pos: Position): Option[IRegion] =
    getNameRegion(pos) flatMap { region =>
      val text = region of sourceFile.content
{'fraction_non_alphanumeric': 0.09866340169370473, 'fraction_numerical': 0.0, 'mean_word_length': 3.9480060575466935, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '265985', 'n_tokens_mistral': 2938, 'n_tokens_neox': 2853, 'n_words': 627}