<commit_before>package eu.stratosphere
package emma.ast

import emma.util._

import cats.Monoid
import cats.implicits._
import shapeless._

import scala.Function.const
import scala.collection.generic.CanBuildFrom
import scala.collection.mutable
import scala.language.higherKinds

/** Utility for traversing and transforming trees. */
trait Transducers { this: AST =>

  import universe._
  import Monoids._

  private def unit[A]: A => Unit =
    const(())

  private def complete[A, R](pf: A =?> R)(args: A)(default: => R): R =
    pf.applyOrElse(args, (_: A) => default)

  /**
   * Placeholder for attribute grammars.
   * @param accumulation Template for accumulated (along the traversal path) attributes.
   * @param inheritance Template for inherited (top-down) attributes.
   * @param synthesis Template for synthesized (bottom-up) attributes.
   * @param doSyn Skip synthesis if not set (saving one pass over the tree).
   * @param MAcc A monoid for accumulated attributes.
   * @param MInh A monoid for inherited attributes.
   * @param MSyn A monoid for synthesized attributes.
   * @tparam Acc The types of accumulated attributes.
   * @tparam Inh The types of inherited attributes.
   * @tparam Syn The types of synthesized attributes.
   */
  case class AttrGrammar[Acc <: HList, Inh <: HList, Syn <: HList](
      accumulation: Attr[Acc,  Inh,  Syn] =?> Acc = PartialFunction.empty,
      inheritance:  Attr[HNil, Inh,  Syn] =?> Inh = PartialFunction.empty,
      synthesis:    Attr[HNil, HNil, Syn] =?> Syn = PartialFunction.empty,
      doSyn: Boolean = false
  )(implicit
    val MAcc: Monoid[Acc],
    val MInh: Monoid[Inh],
    val MSyn: Monoid[Syn]
  ) {

    /** Prepends an accumulated (along the traversal path) attribute. */
    def accumulate[X](acc: Attr[X :: Acc, Inh, Syn] =?> X)(implicit M: Monoid[X]) =
      copy[X :: Acc, Inh, Syn](
        accumulation = { case attr @ Attr(t, _ :: a, i, s)
          if accumulation.isDefinedAt(Attr(t, a, i, s)) || acc.isDefinedAt(attr) =>
            complete(acc)(attr)(M.empty) :: complete(accumulation)(Attr(t, a, i, s))(MAcc.empty)
      })

    /** Prepends an inherited (from parents) attribute. */
    def inherit[X](inh: Attr[HNil, X :: Inh, Syn] =?> X)(implicit M: Monoid[X]) =
      copy[Acc, X :: Inh, Syn](
        accumulation = { case Attr(t, a, _ :: i, s)
          if accumulation.isDefinedAt(Attr(t, a, i, s)) =>
            accumulation(Attr(t, a, i, s))
        }, inheritance = { case attr @ Attr(t, _, _ :: i, s)
          if inheritance.isDefinedAt(Attr(t, i, s)) || inh.isDefinedAt(attr) =>
            complete(inh)(attr)(M.empty) :: complete(inheritance)(Attr(t, i, s))(MInh.empty)
        })

    /** Prepends a synthesized (from children) attribute. */
    def synthesize[X](syn: Attr[HNil, HNil, X :: Syn] =?> X)(implicit M: Monoid[X]) = {
      copy[Acc, Inh, X :: Syn](
        accumulation = { case Attr(t, a, i, _ :: s)
          if accumulation.isDefinedAt(Attr(t, a, i, s)) =>
            accumulation(Attr(t, a, i, s))
        }, inheritance = { case attr @ Attr(t, _, i, _ :: s)
          if inheritance.isDefinedAt(Attr(t, i, s)) =>
            inheritance(Attr(t, i, s))
        }, synthesis = { case attr @ Attr(t, _, _, _ :: s)
          if synthesis.isDefinedAt(Attr(t, s)) || syn.isDefinedAt(attr) =>
            complete(syn)(attr)(M.empty) :: complete(synthesis)(Attr(t, s))(MSyn.empty)
        }, doSyn = true)
    }
  }

  /** Utility for managing attribute grammars. */
  trait ManagedAttr[Acc <: HList, Inh <: HList, Syn <: HList] {

    val grammar: AttrGrammar[Acc, Inh, Syn]
    val callback: Attr[Acc, Inh, Syn] =?> Any  = PartialFunction.empty
    val template: Attr[Acc, Inh, Syn] =?> Tree = PartialFunction.empty

    implicit def MAcc = grammar.MAcc // accumulation monoid
    implicit def MInh = grammar.MInh // inheritance monoid
    implicit def MSyn = grammar.MSyn // synthesis monoid

    /** Accumulated state. */
    protected var state = MAcc.empty

    /** Inherited state. */
    protected var stack = MInh.empty :: Nil

    /** Synthesized state. */
    protected val cache = mutable.Map.empty[Tree, Syn]

    /** Get all accumulated attributes at the current tree. */
    def acc: Acc = state

    /** Get all inherited attributes at the current tree. */
    def inh: Inh = stack.head

    /** Get all synthesized attributes at `tree`. */
    def syn(tree: Tree): Syn =
      if (grammar.doSyn) synMemo(tree)
      else MSyn.empty

    /** Annotates `tree` with all attributes. */
    def ann(tree: Tree): Attr[Acc, Inh, Syn] =
      Attr(tree, acc, inh, syn(tree))

    private lazy val synMemo: Tree => Syn = Memo.recur[Tree, Syn]({ syn => tree =>
      val prev = tree.children.map(syn).foldLeft(MSyn.empty)(MSyn.combine)
      complete(grammar.synthesis)(Attr(tree, prev))(prev)
    }, cache)

    protected lazy val accumulation: Tree =?> Acc = {
      case tree if grammar.accumulation.isDefinedAt(ann(tree)) =>
        grammar.accumulation(ann(tree))<commit_msg>[IR] Fixed a bug in `Transducers.synMemo`.<commit_after>package eu.stratosphere
package emma.ast

import emma.util._

import cats.Monoid
import cats.implicits._
import shapeless._

import scala.Function.const
import scala.collection.generic.CanBuildFrom
import scala.collection.mutable
import scala.language.higherKinds

/** Utility for traversing and transforming trees. */
trait Transducers { this: AST =>

  import universe._
  import Monoids._

  private def unit[A]: A => Unit =
    const(())

  private def complete[A, R](pf: A =?> R)(args: A)(default: => R): R =
    pf.applyOrElse(args, (_: A) => default)

  /**
   * Placeholder for attribute grammars.
   * @param accumulation Template for accumulated (along the traversal path) attributes.
   * @param inheritance Template for inherited (top-down) attributes.
   * @param synthesis Template for synthesized (bottom-up) attributes.
   * @param doSyn Skip synthesis if not set (saving one pass over the tree).
   * @param MAcc A monoid for accumulated attributes.
   * @param MInh A monoid for inherited attributes.
   * @param MSyn A monoid for synthesized attributes.
   * @tparam Acc The types of accumulated attributes.
   * @tparam Inh The types of inherited attributes.
   * @tparam Syn The types of synthesized attributes.
   */
  case class AttrGrammar[Acc <: HList, Inh <: HList, Syn <: HList](
      accumulation: Attr[Acc,  Inh,  Syn] =?> Acc = PartialFunction.empty,
      inheritance:  Attr[HNil, Inh,  Syn] =?> Inh = PartialFunction.empty,
      synthesis:    Attr[HNil, HNil, Syn] =?> Syn = PartialFunction.empty,
      doSyn: Boolean = false
  )(implicit
    val MAcc: Monoid[Acc],
    val MInh: Monoid[Inh],
    val MSyn: Monoid[Syn]
  ) {

    /** Prepends an accumulated (along the traversal path) attribute. */
    def accumulate[X](acc: Attr[X :: Acc, Inh, Syn] =?> X)(implicit M: Monoid[X]) =
      copy[X :: Acc, Inh, Syn](
        accumulation = { case attr @ Attr(t, _ :: a, i, s)
          if accumulation.isDefinedAt(Attr(t, a, i, s)) || acc.isDefinedAt(attr) =>
            complete(acc)(attr)(M.empty) :: complete(accumulation)(Attr(t, a, i, s))(MAcc.empty)
      })

    /** Prepends an inherited (from parents) attribute. */
    def inherit[X](inh: Attr[HNil, X :: Inh, Syn] =?> X)(implicit M: Monoid[X]) =
      copy[Acc, X :: Inh, Syn](
        accumulation = { case Attr(t, a, _ :: i, s)
          if accumulation.isDefinedAt(Attr(t, a, i, s)) =>
            accumulation(Attr(t, a, i, s))
        }, inheritance = { case attr @ Attr(t, _, _ :: i, s)
          if inheritance.isDefinedAt(Attr(t, i, s)) || inh.isDefinedAt(attr) =>
            complete(inh)(attr)(M.empty) :: complete(inheritance)(Attr(t, i, s))(MInh.empty)
        })

    /** Prepends a synthesized (from children) attribute. */
    def synthesize[X](syn: Attr[HNil, HNil, X :: Syn] =?> X)(implicit M: Monoid[X]) = {
      copy[Acc, Inh, X :: Syn](
        accumulation = { case Attr(t, a, i, _ :: s)
          if accumulation.isDefinedAt(Attr(t, a, i, s)) =>
            accumulation(Attr(t, a, i, s))
        }, inheritance = { case attr @ Attr(t, _, i, _ :: s)
          if inheritance.isDefinedAt(Attr(t, i, s)) =>
            inheritance(Attr(t, i, s))
        }, synthesis = { case attr @ Attr(t, _, _, _ :: s)
          if synthesis.isDefinedAt(Attr(t, s)) || syn.isDefinedAt(attr) =>
            complete(syn)(attr)(M.empty) :: complete(synthesis)(Attr(t, s))(MSyn.empty)
        }, doSyn = true)
    }
  }

  /** Utility for managing attribute grammars. */
  trait ManagedAttr[Acc <: HList, Inh <: HList, Syn <: HList] {

    val grammar: AttrGrammar[Acc, Inh, Syn]
    val callback: Attr[Acc, Inh, Syn] =?> Any  = PartialFunction.empty
    val template: Attr[Acc, Inh, Syn] =?> Tree = PartialFunction.empty

    implicit def MAcc = grammar.MAcc // accumulation monoid
    implicit def MInh = grammar.MInh // inheritance monoid
    implicit def MSyn = grammar.MSyn // synthesis monoid

    /** Accumulated state. */
    protected var state = MAcc.empty

    /** Inherited state. */
    protected var stack = MInh.empty :: Nil

    /** Synthesized state. */
    protected val cache = mutable.Map.empty[Tree, Syn]

    /** Get all accumulated attributes at the current tree. */
    def acc: Acc = state

    /** Get all inherited attributes at the current tree. */
    def inh: Inh = stack.head

    /** Get all synthesized attributes at `tree`. */
    def syn(tree: Tree): Syn =
      if (grammar.doSyn) synMemo(tree)
      else MSyn.empty

    /** Annotates `tree` with all attributes. */
    def ann(tree: Tree): Attr[Acc, Inh, Syn] =
      Attr(tree, acc, inh, syn(tree))

    private lazy val synMemo: Tree => Syn = Memo.recur[Tree, Syn]({ syn => tree =>
      val prev = tree.children.map(syn).foldLeft(MSyn.empty)(MSyn.combine)
      val curr = complete(grammar.synthesis)(Attr(tree, prev))(MSyn.empty)
      MSyn.combine(prev, curr)
    }, cache)

    protected lazy val accumulation: Tree =?> Acc = {
      case tree if grammar.accumulation.isDefinedAt(ann(tree)) =>
        grammar.accumulation(ann(tree))
{'fraction_non_alphanumeric': 0.14213499851323225, 'fraction_numerical': 0.0, 'mean_word_length': 2.9646365422396856, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '832268', 'n_tokens_mistral': 3457, 'n_tokens_neox': 3281, 'n_words': 954}