<commit_before>package scala.meta
package internal
package ast

import scala.{Seq => _}
import scala.collection.immutable.Seq
import scala.compat.Platform.EOL
import scala.reflect.{classTag, ClassTag}
import scala.meta.internal.ast._
import scala.meta.internal.ast.Helpers._
import scala.meta.internal.semantic._
import scala.meta.internal.semantic.RuntimeConverters._
import scala.meta.internal.prettyprinters._
import org.scalameta.collections._
import org.scalameta.invariants._
import org.scalameta.unreachable
import org.scalameta.debug._
import scala.meta.internal.{ast => m}
import scala.meta.prettyprinters._

object mergeTrees {
  // NOTE: "sy-" stands for "syntactic", "se-" stands for "semantic", "me-" stands for "merged".s
  // NOTE: Much like in LogicalTrees and in ToMtree, cases here must be ordered according
  // to the order of appearance of the corresponding AST nodes in Trees.scala.
  // TODO: I think we could hardcode this traversal into the @ast infrastructure.
  // That will let us save time on: 1) eager recreation of entire trees, 2) eager computation of tokens for withTokens.
  def apply[T <: Tree : ClassTag](sy: T, se: Tree): T = {
    object loop {
      def apply[T <: Tree : ClassTag](sy1: T, se1: Tree): T = {
        // NOTE: This roundabout way of recursing is here only for error reporting.
        // If we end up in, say, `apply(syss, sess)`, and we have `List()` vs `List(List())`,
        // then we will have troubles providing a nice traceback to the user (because we don't have parents at hand).
        // There are two solutions to this, as far as I can see:
        // 1) Pass parents explicitly to all calls to `loop` that deal with collections.
        // 2) Have `loop` definitions be nested within `mergeTrees.apply`, so that everyone can access `sy` and `se`.
        // I tried #1, and it's way too much hassle, which is why I settled down on #2.
        if ((sy1 ne sy) || (se1 ne se)) mergeTrees(sy1, se1)
        else {
          val expandedMetree = (sy, se) match {
            // ============ NAMES ============

            case (sy: m.Name.Anonymous, se: m.Name.Anonymous) =>
              sy.copy()
            case (sy: m.Name.Indeterminate, se: m.Name.Indeterminate) =>
              sy.copy()

            // ============ TERMS ============

            case (sy: m.Term.This, se: m.Term.This) =>
              sy.copy(loop(sy.qual, se.qual))
            case (sy: m.Term.Name, se: m.Term.Name) =>
              sy.copy()
            case (sy: m.Term.Apply, se: m.Term.Apply) =>
              sy.copy(loop(sy.fun, se.fun), loop(sy.args, se.args))
            case (sy: m.Term.ApplyInfix, se @ m.Term.Apply(sefun, seargs)) =>
              val (selhs, seop, setargs) = sefun match {
                case m.Term.Select(selhs, seop) => (selhs, seop, Nil)
                case m.Type.Apply(m.Term.Select(selhs, seop), setargs) => (selhs, seop, setargs)
              }
              require(seop.isLeftAssoc && debug(sy, se)) // TODO: right-associative operators aren't supported yet
              sy.copy(loop(sy.lhs, selhs), loop(sy.op, seop), loop(sy.targs, setargs), loop(sy.args, seargs))
            case (sy: m.Term.ApplyType, se: m.Term.ApplyType) =>
              sy.copy(loop(sy.fun, se.fun), loop(sy.targs, se.targs))
            case (sy: m.Term.Param, se: m.Term.Param) =>
              sy.copy(loop(sy.mods, se.mods), loop(sy.name, se.name), loop(sy.decltpe, se.decltpe), loop(sy.default, se.default))

            // ============ TYPES ============

            case (sy: m.Type.Name, se: m.Type.Name) =>
              sy.copy()
            case (sy: m.Type.Select, se: m.Type.Select) =>
              sy.copy(loop(sy.qual, se.qual), loop(sy.name, se.name))
            case (sy: m.Type.Apply, se: m.Type.Apply) =>
              sy.copy(loop(sy.tpe, se.tpe), loop(sy.args, se.args))

            // ============ PATTERNS ============

            // ============ LITERALS ============

            case (sy: m.Lit, se: m.Lit) =>
              sy.copy()

            // ============ DECLS ============

            // ============ DEFNS ============

            case (sy: m.Defn.Def, se: m.Defn.Def) =>
              if (sy.name.toString != se.name.toString) failCorrelate(sy, se, "incompatible methods")
              val medecltpe = (sy.decltpe, se.decltpe) match {
                case (None, Some(se)) => None
                case (sy, se) => loop(sy, se)
              }
              sy.copy(loop(sy.mods, se.mods), loop(sy.name, se.name), loop(sy.tparams, se.tparams), loop(sy.paramss, se.paramss), medecltpe, loop(sy.body, se.body))
            case (sy: m.Defn.Class, se: m.Defn.Class) =>
              if (sy.name.toString != se.name.toString) failCorrelate(sy, se, "incompatible classes")
              sy.copy(loop(sy.mods, se.mods), loop(sy.name, se.name), loop(sy.tparams, se.tparams), loop(sy.ctor, se.ctor), loop(sy.templ, se.templ))

            // ============ PKGS ============

            case (sy: m.Source, se: m.Source) =>
              sy.copy(loop(sy.stats, se.stats))
            case (sy: m.Pkg, se: m.Pkg) =>
              if (sy.ref.toString != se.ref.toString) failCorrelate(sy, se, "incompatible packages")
              sy.copy(loop(sy.ref, se.ref), loop(sy.stats, se.stats))

            // ============ CTORS ============

            case (sy: m.Ctor.Primary, se: m.Ctor.Primary) =>
              // NOTE: scala.reflect irreversibly desugars nullary constructors into empty-arglist ones
              val meparamss = (sy.paramss, se.paramss) match {
                case (Seq(), Seq(Seq())) => List()
                case (syss, sess) => loop(syss, sess)
              }
              sy.copy(loop(sy.mods, se.mods), loop(sy.name, se.name), meparamss)
            case (sy: m.Ctor.Ref.Name, se: m.Ctor.Ref.Name) =>
              sy.copy()
            case (sy: m.Ctor.Ref.Select, se: m.Ctor.Ref.Select) =>
              sy.copy(loop(sy.qual, se.qual), loop(sy.name, se.name))

            // ============ TEMPLATES ============

            case (sy: m.Template, se: m.Template) =>
              // NOTE: ensugaring rules for parent lists, as per Scala 2.11.7
              // 1) If the parent list is empty, make it List(AnyRef)
              // 2) If parsing a case class, append Product and Serializable to the end of the parent list
              // 3) During typechecking filter out all subsequent repeated occurrences of ProductN, Product and Serializable
              // 4) If the first parent in the list is a trait, then:
              //    * Convert it to AnyRef, if it's Any
              //    * Prepend tpe.firstParent to the list, otherwise
              // 5) If a parent is applied to a nullary argument list, make it empty argument list.
              def mergeParents(sys: Seq[m.Ctor.Call], ses: Seq[m.Ctor.Call]): Seq[m.Ctor.Call] = {
                if (sys.length != ses.length) failCorrelate(sy, se, sys, ses)
                sys.zip(ses).map({
                  case (sy, m.Term.Apply(se, Nil)) => loop(sy, se)
                  case (sy, se) => loop(sy, se)
                })
              }
              val meparents = (sy.parents, se.parents) match {
                case (Seq(), Seq(m.Term.Apply(anyRef: m.Ctor.Ref, Nil))) if anyRef.refersTo(Object_init) =>
                  List()
                case (syss, sess) =>
                  mergeParents(syss, sess)
              }
              val mestats = (sy.stats, se.stats) match {
                case (None, Some(Nil)) => None
                case (sys, ses) => loop(sys, ses)
              }
              sy.copy(loop(sy.early, se.early), meparents, loop(sy.self, se.self), mestats)

            // ============ MODIFIERS ============

            // ============ ODDS & ENDS ============

            case _ =>
              failCorrelate(sy, se, "unexpected trees")
          }

          val metree = expandedMetree.withTokens(sy.tokens).inheritAttrs(se).withTypechecked(se.isTypechecked)<commit_msg>documents mergeTrees a little bit<commit_after>package scala.meta
package internal
package ast

import scala.{Seq => _}
import scala.collection.immutable.Seq
import scala.compat.Platform.EOL
import scala.reflect.{classTag, ClassTag}
import scala.meta.internal.ast._
import scala.meta.internal.ast.Helpers._
import scala.meta.internal.semantic._
import scala.meta.internal.semantic.RuntimeConverters._
import scala.meta.internal.prettyprinters._
import org.scalameta.collections._
import org.scalameta.invariants._
import org.scalameta.unreachable
import org.scalameta.debug._
import scala.meta.internal.{ast => m}
import scala.meta.prettyprinters._

// NOTE: This facility is used to merge syntactic trees (carrying perfect syntactic information)
// and semantic trees (carrying comprehensive set of semantic attributes for themselves and their subnodes).
// "sy-" stands for "syntactic", "se-" stands for "semantic", "me-" stands for "merged".
//
// The main difficulty here is supporting different desugarings that Scala typecheckers may perform.
// Since mergeTrees is an essential part of deserialization from TASTY, this needs to be platform-independent,
// so we have to support both scalac and dotc (also, as discussed with @smarter, it's possible to make mergeTrees
// platform-dependent, but we'll have to pay the price of user convenience).
//
// Here are the desugarings that we support
// (S stands for Scala 2.11.x, D stands for Dotty):
//   1) (S) Inference of method return types
//   2) (S) Desugaring of nullary constructors to empty-paramlist constructors
//   3) (S) Appending AnyRef to the end of an empty parent list
//   4) (S) Appending Product and Serializable to the end of the parent list of a case class
//   5) (S) Weeding out repeated occurrences of ProductN, Product and Serializable from the parent list
//   6) (S) Converting Any to AnyRef in the parent list that starts with a Any
//   7) (S) Prepending tpe.firstParent to the parent list that starts with a trait different from Any
//   8) (S) Converting nullary parents to empty-arglist parents
object mergeTrees {
  // NOTE: Much like in LogicalTrees and in ToMtree, cases here must be ordered according
  // to the order of appearance of the corresponding AST nodes in Trees.scala.
  // TODO: I think we could hardcode this traversal into the @ast infrastructure.
  // That will let us save time on: 1) eager recreation of entire trees, 2) eager computation of tokens for withTokens.
  def apply[T <: Tree : ClassTag](sy: T, se: Tree): T = {
    object loop {
      def apply[T <: Tree : ClassTag](sy1: T, se1: Tree): T = {
        // NOTE: This roundabout way of recursing is here only for error reporting.
        // If we end up in, say, `apply(syss, sess)`, and we have `List()` vs `List(List())`,
        // then we will have troubles providing a nice traceback to the user (because we don't have parents at hand).
        // There are two solutions to this, as far as I can see:
        // 1) Pass parents explicitly to all calls to `loop` that deal with collections.
        // 2) Have `loop` definitions be nested within `mergeTrees.apply`, so that everyone can access `sy` and `se`.
        // I tried #1, and it's way too much hassle, which is why I settled down on #2.
        if ((sy1 ne sy) || (se1 ne se)) mergeTrees(sy1, se1)
        else {
          require(se.isTypechecked)
          val expandedMetree = (sy, se) match {
            // ============ NAMES ============

            case (sy: m.Name.Anonymous, se: m.Name.Anonymous) =>
              sy.copy()
            case (sy: m.Name.Indeterminate, se: m.Name.Indeterminate) =>
              sy.copy()

            // ============ TERMS ============

            case (sy: m.Term.This, se: m.Term.This) =>
              sy.copy(loop(sy.qual, se.qual))
            case (sy: m.Term.Name, se: m.Term.Name) =>
              sy.copy()
            case (sy: m.Term.Apply, se: m.Term.Apply) =>
              sy.copy(loop(sy.fun, se.fun), loop(sy.args, se.args))
            case (sy: m.Term.ApplyInfix, se @ m.Term.Apply(sefun, seargs)) =>
              val (selhs, seop, setargs) = sefun match {
                case m.Term.Select(selhs, seop) => (selhs, seop, Nil)
                case m.Type.Apply(m.Term.Select(selhs, seop), setargs) => (selhs, seop, setargs)
              }
              require(seop.isLeftAssoc && debug(sy, se)) // TODO: right-associative operators aren't supported yet
              sy.copy(loop(sy.lhs, selhs), loop(sy.op, seop), loop(sy.targs, setargs), loop(sy.args, seargs))
            case (sy: m.Term.ApplyType, se: m.Term.ApplyType) =>
              sy.copy(loop(sy.fun, se.fun), loop(sy.targs, se.targs))
            case (sy: m.Term.Param, se: m.Term.Param) =>
              sy.copy(loop(sy.mods, se.mods), loop(sy.name, se.name), loop(sy.decltpe, se.decltpe), loop(sy.default, se.default))

            // ============ TYPES ============

            case (sy: m.Type.Name, se: m.Type.Name) =>
              sy.copy()
            case (sy: m.Type.Select, se: m.Type.Select) =>
              sy.copy(loop(sy.qual, se.qual), loop(sy.name, se.name))
            case (sy: m.Type.Apply, se: m.Type.Apply) =>
              sy.copy(loop(sy.tpe, se.tpe), loop(sy.args, se.args))

            // ============ PATTERNS ============

            // ============ LITERALS ============

            case (sy: m.Lit, se: m.Lit) =>
              sy.copy()

            // ============ DECLS ============

            // ============ DEFNS ============

            case (sy: m.Defn.Def, se: m.Defn.Def) =>
              if (sy.name.toString != se.name.toString) failCorrelate(sy, se, "incompatible methods")
              val medecltpe = (sy.decltpe, se.decltpe) match { // (1)
                case (None, Some(se)) => None
                case (sy, se) => loop(sy, se)
              }
              sy.copy(loop(sy.mods, se.mods), loop(sy.name, se.name), loop(sy.tparams, se.tparams), loop(sy.paramss, se.paramss), medecltpe, loop(sy.body, se.body))
            case (sy: m.Defn.Class, se: m.Defn.Class) =>
              if (sy.name.toString != se.name.toString) failCorrelate(sy, se, "incompatible classes")
              sy.copy(loop(sy.mods, se.mods), loop(sy.name, se.name), loop(sy.tparams, se.tparams), loop(sy.ctor, se.ctor), loop(sy.templ, se.templ))

            // ============ PKGS ============

            case (sy: m.Source, se: m.Source) =>
              sy.copy(loop(sy.stats, se.stats))
            case (sy: m.Pkg, se: m.Pkg) =>
              if (sy.ref.toString != se.ref.toString) failCorrelate(sy, se, "incompatible packages")
              sy.copy(loop(sy.ref, se.ref), loop(sy.stats, se.stats))

            // ============ CTORS ============

            case (sy: m.Ctor.Primary, se: m.Ctor.Primary) =>
              val meparamss = (sy.paramss, se.paramss) match { // (2)
                case (Seq(), Seq(Seq())) => List()
                case (syss, sess) => loop(syss, sess)
              }
              sy.copy(loop(sy.mods, se.mods), loop(sy.name, se.name), meparamss)
            case (sy: m.Ctor.Ref.Name, se: m.Ctor.Ref.Name) =>
              sy.copy()
            case (sy: m.Ctor.Ref.Select, se: m.Ctor.Ref.Select) =>
              sy.copy(loop(sy.qual, se.qual), loop(sy.name, se.name))

            // ============ TEMPLATES ============

            case (sy: m.Template, se: m.Template) =>
              def mergeParents(sys: Seq[m.Ctor.Call], ses: Seq[m.Ctor.Call]): Seq[m.Ctor.Call] = {
                if (sys.length != ses.length) failCorrelate(sy, se, sys, ses)
                sys.zip(ses).map({ // (8)
                  case (sy, m.Term.Apply(se, Nil)) => loop(sy, se)
                  case (sy, se) => loop(sy, se)
                })
              }
              val meparents = (sy.parents, se.parents) match {
                case (Seq(), Seq(m.Term.Apply(anyRef: m.Ctor.Ref, Nil))) if anyRef.refersTo(Object_init) => // (3)
                  List()
                case (syss, sess) =>
                  mergeParents(syss, sess)
              }
              val mestats = (sy.stats, se.stats) match {
                case (None, Some(Nil)) => None
                case (sys, ses) => loop(sys, ses)
              }
              sy.copy(loop(sy.early, se.early), meparents, loop(sy.self, se.self), mestats)

            // ============ MODIFIERS ============

            // ============ ODDS & ENDS ============

            case _ =>
              failCorrelate(sy, se, "unexpected trees")
          }

          val metree = expandedMetree.withTokens(sy.tokens).inheritAttrs(se).withTypechecked(se.isTypechecked)
{'fraction_non_alphanumeric': 0.16411457395063953, 'fraction_numerical': 0.002882363538101243, 'mean_word_length': 2.1198950917946795, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5326955', 'n_tokens_mistral': 5393, 'n_tokens_neox': 5181, 'n_words': 1478}