<commit_before>package dotty.tools
package dotc
package core
package pickling

import ast.Trees._
import PickleFormat._
import core._
import Contexts._, Symbols._, Types._, Names._, Constants._, Decorators._, Annotations._, StdNames.tpnme
import collection.mutable
import TastyBuffer._

class TreePickler(pickler: TastyPickler) {
  val buf = new TreeBuffer
  pickler.newSection("ASTs", buf)
  import buf._
  import pickler.nameBuffer.nameIndex
  import ast.tpd._

  private val symRefs = new mutable.HashMap[Symbol, Addr]
  private val forwardSymRefs = new mutable.HashMap[Symbol, List[Addr]]
  private val pickledTypes = new java.util.IdentityHashMap[Type, Any] // Value type is really Addr, but that's not compatible with null

  private def withLength(op: => Unit) = {
    val lengthAddr = reserveRef(relative = true)
    op
    fillRef(lengthAddr, currentAddr, relative = true)
  }
  
  private var makeSymbolicRefsTo: Symbol = NoSymbol

  /** All references to members of class `sym` are pickled
   *  as symbolic references. Used to pickle the self info of a class.
   *  Without this precaution we get an infinite cycle when unpickling pos/extmethods.scala
   *  The problem arises when a self type of a trait is a type parameter of the same trait.
   */
  private def withSymbolicRefsTo[T](sym: Symbol)(op: => T): T = {
    val saved = makeSymbolicRefsTo
    makeSymbolicRefsTo = sym
    try op
    finally makeSymbolicRefsTo = saved
  }
  
  def preRegister(tree: Tree)(implicit ctx: Context): Unit = tree match {
    case tree: MemberDef => 
      if (!symRefs.contains(tree.symbol)) symRefs(tree.symbol) = NoAddr
    case _ =>
  }

  def registerDef(sym: Symbol): Unit = {
    symRefs(sym) = currentAddr
    forwardSymRefs.get(sym) match {
      case Some(refs) =>
        refs.foreach(fillRef(_, currentAddr, relative = false))
        forwardSymRefs -= sym
      case None =>
    }
  }

  private def pickleName(name: Name) = writeNat(nameIndex(name).index)
  private def pickleName(name: TastyName) = writeNat(nameIndex(name).index)
  private def pickleNameAndSig(name: Name, sig: Signature) = {
    val Signature(params, result) = sig
    pickleName(TastyName.Signed(nameIndex(name), params.map(nameIndex), nameIndex(result)))
  }

  private def pickleSymRef(sym: Symbol)(implicit ctx: Context) = symRefs.get(sym) match {
    case Some(label) =>
      if (label != NoAddr) writeRef(label) else pickleForwardSymRef(sym)
    case None =>
      ctx.log(i"pickling reference to as yet undefined $sym in ${sym.owner}", sym.pos)
      pickleForwardSymRef(sym)
  }
  
  private def pickleForwardSymRef(sym: Symbol)(implicit ctx: Context) = {
    val ref = reserveRef(relative = false)
    assert(!sym.is(Flags.Package), sym)
    forwardSymRefs(sym) = ref :: forwardSymRefs.getOrElse(sym, Nil)  
  }

  def pickle(trees: List[Tree])(implicit ctx: Context) = {
    
    def qualifiedName(sym: Symbol): TastyName =
      if (sym.isRoot || sym.owner.isRoot) TastyName.Simple(sym.name.toTermName)
      else TastyName.Qualified(nameIndex(qualifiedName(sym.owner)), nameIndex(sym.name))

    def pickleConstant(c: Constant): Unit = c.tag match {
      case UnitTag =>
        writeByte(UNITconst)
      case BooleanTag =>
        writeByte(if (c.booleanValue) TRUEconst else FALSEconst)
      case ByteTag =>
        writeByte(BYTEconst)
        writeInt(c.byteValue)
      case ShortTag =>
        writeByte(SHORTconst)
        writeInt(c.shortValue)
      case CharTag =>
        writeByte(CHARconst)
        writeNat(c.charValue)
      case IntTag =>
        writeByte(INTconst)
        writeInt(c.intValue)
      case LongTag =>
        writeByte(LONGconst)
        writeLongInt(c.longValue)
      case FloatTag =>
        writeByte(FLOATconst)
        writeInt(java.lang.Float.floatToRawIntBits(c.floatValue))
      case DoubleTag =>
        writeByte(DOUBLEconst)
        writeLongInt(java.lang.Double.doubleToRawLongBits(c.doubleValue))
      case StringTag =>
        writeByte(STRINGconst)
        writeNat(nameIndex(c.stringValue).index)
      case NullTag =>
        writeByte(NULLconst)
      case ClazzTag =>
        writeByte(CLASSconst)
        pickleType(c.typeValue)
      case EnumTag =>
        writeByte(ENUMconst)
        pickleType(c.symbolValue.termRef)
    }

    def pickleType(tpe0: Type, richTypes: Boolean = false): Unit = try {
      val tpe = tpe0.stripTypeVar
      val prev = pickledTypes.get(tpe)
      if (prev == null) {
        pickledTypes.put(tpe, currentAddr)
        pickleNewType(tpe, richTypes)
      }
      else {
        writeByte(SHARED)
        writeRef(prev.asInstanceOf[Addr])
      }
    } catch {
      case ex: AssertionError =>
        println(i"error when pickling type $tpe0")
        throw ex
    }
      
    def pickleNewType(tpe: Type, richTypes: Boolean): Unit = try { tpe match {
      case ConstantType(value) => 
        pickleConstant(value)
      case tpe: TypeRef if tpe.info.isAlias && tpe.symbol.is(Flags.AliasPreferred) =>
        pickleType(tpe.info.bounds.hi)
      case tpe: WithFixedSym =>
        val sym = tpe.symbol
        if (sym.is(Flags.Package)) {
          writeByte(if (tpe.isType) TYPEREFpkg else TERMREFpkg)
          pickleName(qualifiedName(sym))
        } 
        else if (tpe.prefix == NoPrefix) {
          def pickleRef() = {
            writeByte(if (tpe.isType) TYPEREFdirect else TERMREFdirect)
            pickleSymRef(sym)        
          }
          if (sym is Flags.BindDefinedType) {
            registerDef(sym)
            writeByte(BIND)
            withLength {
              pickleName(sym.name)
              pickleType(sym.info)
              pickleRef()
            }
          }
          else pickleRef()
        }
        else {
          writeByte(if (tpe.isType) TYPEREFsymbol else TERMREFsymbol)
          pickleSymRef(sym); pickleType(tpe.prefix)
        }
      case tpe: TermRefWithSignature =>
        writeByte(TERMREF)
        pickleNameAndSig(tpe.name, tpe.signature); pickleType(tpe.prefix)
      case tpe: NamedType =>
        if (tpe.name == tpnme.Apply && tpe.prefix.argInfos.nonEmpty && tpe.prefix.isInstantiatedLambda)
          // instantiated lambdas are pickled as APPLIEDTYPE; #Apply will 
          // be reconstituted when unpickling.
          pickleType(tpe.prefix)
        else tpe.prefix match {
          case prefix: ThisType if prefix.cls == makeSymbolicRefsTo =>
            pickleType(NamedType.withFixedSym(tpe.prefix, tpe.symbol))
          case _ =>
            writeByte(if (tpe.isType) TYPEREF else TERMREF)
            pickleName(tpe.name); pickleType(tpe.prefix)
        }
      case tpe: ThisType =>
        writeByte(THIS)
        pickleType(tpe.tref)
      case tpe: SuperType =>
        writeByte(SUPERtype)
        withLength { pickleType(tpe.thistpe); pickleType(tpe.supertpe)}
      case tpe: SkolemType =>
        writeByte(SKOLEMtype)
        writeRef(pickledTypes.get(tpe.binder).asInstanceOf[Addr])
      case tpe: RefinedType =>
        val args = tpe.argInfos(interpolate = true)
        if (args.isEmpty) {
          writeByte(REFINEDtype)
          withLength { 
            pickleType(tpe.parent)
            pickleName(tpe.refinedName)
            pickleType(tpe.refinedInfo, richTypes = true) 
          }
        }
        else {
          writeByte(APPLIEDtype)
          withLength { pickleType(tpe.withoutArgs(args)); args.foreach(pickleType(_)) }
        }
      case tpe: TypeAlias =>
        writeByte(TYPEALIAS)
        withLength { 
          pickleType(tpe.alias, richTypes) 
          tpe.variance match {
            case 1 => writeByte(COVARIANT)
            case -1 => writeByte(CONTRAVARIANT)
            case 0 =>
          }<commit_msg>Revert of interpolation decision when generating APPLIEDTYPEs<commit_after>package dotty.tools
package dotc
package core
package pickling

import ast.Trees._
import PickleFormat._
import core._
import Contexts._, Symbols._, Types._, Names._, Constants._, Decorators._, Annotations._, StdNames.tpnme
import collection.mutable
import TastyBuffer._

class TreePickler(pickler: TastyPickler) {
  val buf = new TreeBuffer
  pickler.newSection("ASTs", buf)
  import buf._
  import pickler.nameBuffer.nameIndex
  import ast.tpd._

  private val symRefs = new mutable.HashMap[Symbol, Addr]
  private val forwardSymRefs = new mutable.HashMap[Symbol, List[Addr]]
  private val pickledTypes = new java.util.IdentityHashMap[Type, Any] // Value type is really Addr, but that's not compatible with null

  private def withLength(op: => Unit) = {
    val lengthAddr = reserveRef(relative = true)
    op
    fillRef(lengthAddr, currentAddr, relative = true)
  }
  
  private var makeSymbolicRefsTo: Symbol = NoSymbol

  /** All references to members of class `sym` are pickled
   *  as symbolic references. Used to pickle the self info of a class.
   *  Without this precaution we get an infinite cycle when unpickling pos/extmethods.scala
   *  The problem arises when a self type of a trait is a type parameter of the same trait.
   */
  private def withSymbolicRefsTo[T](sym: Symbol)(op: => T): T = {
    val saved = makeSymbolicRefsTo
    makeSymbolicRefsTo = sym
    try op
    finally makeSymbolicRefsTo = saved
  }
  
  def preRegister(tree: Tree)(implicit ctx: Context): Unit = tree match {
    case tree: MemberDef => 
      if (!symRefs.contains(tree.symbol)) symRefs(tree.symbol) = NoAddr
    case _ =>
  }

  def registerDef(sym: Symbol): Unit = {
    symRefs(sym) = currentAddr
    forwardSymRefs.get(sym) match {
      case Some(refs) =>
        refs.foreach(fillRef(_, currentAddr, relative = false))
        forwardSymRefs -= sym
      case None =>
    }
  }

  private def pickleName(name: Name) = writeNat(nameIndex(name).index)
  private def pickleName(name: TastyName) = writeNat(nameIndex(name).index)
  private def pickleNameAndSig(name: Name, sig: Signature) = {
    val Signature(params, result) = sig
    pickleName(TastyName.Signed(nameIndex(name), params.map(nameIndex), nameIndex(result)))
  }

  private def pickleSymRef(sym: Symbol)(implicit ctx: Context) = symRefs.get(sym) match {
    case Some(label) =>
      if (label != NoAddr) writeRef(label) else pickleForwardSymRef(sym)
    case None =>
      ctx.log(i"pickling reference to as yet undefined $sym in ${sym.owner}", sym.pos)
      pickleForwardSymRef(sym)
  }
  
  private def pickleForwardSymRef(sym: Symbol)(implicit ctx: Context) = {
    val ref = reserveRef(relative = false)
    assert(!sym.is(Flags.Package), sym)
    forwardSymRefs(sym) = ref :: forwardSymRefs.getOrElse(sym, Nil)  
  }

  def pickle(trees: List[Tree])(implicit ctx: Context) = {
    
    def qualifiedName(sym: Symbol): TastyName =
      if (sym.isRoot || sym.owner.isRoot) TastyName.Simple(sym.name.toTermName)
      else TastyName.Qualified(nameIndex(qualifiedName(sym.owner)), nameIndex(sym.name))

    def pickleConstant(c: Constant): Unit = c.tag match {
      case UnitTag =>
        writeByte(UNITconst)
      case BooleanTag =>
        writeByte(if (c.booleanValue) TRUEconst else FALSEconst)
      case ByteTag =>
        writeByte(BYTEconst)
        writeInt(c.byteValue)
      case ShortTag =>
        writeByte(SHORTconst)
        writeInt(c.shortValue)
      case CharTag =>
        writeByte(CHARconst)
        writeNat(c.charValue)
      case IntTag =>
        writeByte(INTconst)
        writeInt(c.intValue)
      case LongTag =>
        writeByte(LONGconst)
        writeLongInt(c.longValue)
      case FloatTag =>
        writeByte(FLOATconst)
        writeInt(java.lang.Float.floatToRawIntBits(c.floatValue))
      case DoubleTag =>
        writeByte(DOUBLEconst)
        writeLongInt(java.lang.Double.doubleToRawLongBits(c.doubleValue))
      case StringTag =>
        writeByte(STRINGconst)
        writeNat(nameIndex(c.stringValue).index)
      case NullTag =>
        writeByte(NULLconst)
      case ClazzTag =>
        writeByte(CLASSconst)
        pickleType(c.typeValue)
      case EnumTag =>
        writeByte(ENUMconst)
        pickleType(c.symbolValue.termRef)
    }

    def pickleType(tpe0: Type, richTypes: Boolean = false): Unit = try {
      val tpe = tpe0.stripTypeVar
      val prev = pickledTypes.get(tpe)
      if (prev == null) {
        pickledTypes.put(tpe, currentAddr)
        pickleNewType(tpe, richTypes)
      }
      else {
        writeByte(SHARED)
        writeRef(prev.asInstanceOf[Addr])
      }
    } catch {
      case ex: AssertionError =>
        println(i"error when pickling type $tpe0")
        throw ex
    }
      
    def pickleNewType(tpe: Type, richTypes: Boolean): Unit = try { tpe match {
      case ConstantType(value) => 
        pickleConstant(value)
      case tpe: TypeRef if tpe.info.isAlias && tpe.symbol.is(Flags.AliasPreferred) =>
        pickleType(tpe.info.bounds.hi)
      case tpe: WithFixedSym =>
        val sym = tpe.symbol
        if (sym.is(Flags.Package)) {
          writeByte(if (tpe.isType) TYPEREFpkg else TERMREFpkg)
          pickleName(qualifiedName(sym))
        } 
        else if (tpe.prefix == NoPrefix) {
          def pickleRef() = {
            writeByte(if (tpe.isType) TYPEREFdirect else TERMREFdirect)
            pickleSymRef(sym)        
          }
          if (sym is Flags.BindDefinedType) {
            registerDef(sym)
            writeByte(BIND)
            withLength {
              pickleName(sym.name)
              pickleType(sym.info)
              pickleRef()
            }
          }
          else pickleRef()
        }
        else {
          writeByte(if (tpe.isType) TYPEREFsymbol else TERMREFsymbol)
          pickleSymRef(sym); pickleType(tpe.prefix)
        }
      case tpe: TermRefWithSignature =>
        writeByte(TERMREF)
        pickleNameAndSig(tpe.name, tpe.signature); pickleType(tpe.prefix)
      case tpe: NamedType =>
        if (tpe.name == tpnme.Apply && tpe.prefix.argInfos.nonEmpty && tpe.prefix.isInstantiatedLambda)
          // instantiated lambdas are pickled as APPLIEDTYPE; #Apply will 
          // be reconstituted when unpickling.
          pickleType(tpe.prefix)
        else tpe.prefix match {
          case prefix: ThisType if prefix.cls == makeSymbolicRefsTo =>
            pickleType(NamedType.withFixedSym(tpe.prefix, tpe.symbol))
          case _ =>
            writeByte(if (tpe.isType) TYPEREF else TERMREF)
            pickleName(tpe.name); pickleType(tpe.prefix)
        }
      case tpe: ThisType =>
        writeByte(THIS)
        pickleType(tpe.tref)
      case tpe: SuperType =>
        writeByte(SUPERtype)
        withLength { pickleType(tpe.thistpe); pickleType(tpe.supertpe)}
      case tpe: SkolemType =>
        writeByte(SKOLEMtype)
        writeRef(pickledTypes.get(tpe.binder).asInstanceOf[Addr])
      case tpe: RefinedType =>
        val args = tpe.argInfos(interpolate = false)
        if (args.isEmpty) {
          writeByte(REFINEDtype)
          withLength { 
            pickleType(tpe.parent)
            pickleName(tpe.refinedName)
            pickleType(tpe.refinedInfo, richTypes = true) 
          }
        }
        else {
          writeByte(APPLIEDtype)
          withLength { pickleType(tpe.withoutArgs(args)); args.foreach(pickleType(_)) }
        }
      case tpe: TypeAlias =>
        writeByte(TYPEALIAS)
        withLength { 
          pickleType(tpe.alias, richTypes) 
          tpe.variance match {
            case 1 => writeByte(COVARIANT)
            case -1 => writeByte(CONTRAVARIANT)
            case 0 =>
          }
{'fraction_non_alphanumeric': 0.09859972897980254, 'fraction_numerical': 0.0007743434213073498, 'mean_word_length': 2.5375485049075555, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1785935', 'n_tokens_mistral': 4935, 'n_tokens_neox': 4850, 'n_words': 1074}