<commit_before>package eu.stratosphere.emma
package compiler

import ast.AST

import org.scalactic._
import org.scalactic.Accumulation._

/** Common IR tools. */
trait Common extends ReflectUtil with AST {

  import universe._

  // --------------------------------------------------------------------------
  // Emma API
  // --------------------------------------------------------------------------

  /** A set of API method symbols to be comprehended. */
  protected[emma] object API {
    val moduleSymbol /*    */ = rootMirror.staticModule("eu.stratosphere.emma.api.package")
    val bagSymbol /*       */ = rootMirror.staticClass("eu.stratosphere.emma.api.DataBag")
    val groupSymbol /*     */ = rootMirror.staticClass("eu.stratosphere.emma.api.Group")
    val statefulSymbol /*  */ = rootMirror.staticClass("eu.stratosphere.emma.api.Stateful.Bag")
    val inputFmtSymbol /*  */ = rootMirror.staticClass("eu.stratosphere.emma.api.InputFormat")
    val outputFmtSymbol /* */ = rootMirror.staticClass("eu.stratosphere.emma.api.OutputFormat")

    val apply /*           */ = bagSymbol.companion.info.decl(TermName("apply"))
    val read /*            */ = moduleSymbol.info.decl(TermName("read"))
    val write /*           */ = moduleSymbol.info.decl(TermName("write"))
    val stateful /*        */ = moduleSymbol.info.decl(TermName("stateful"))
    val fold /*            */ = bagSymbol.info.decl(TermName("fold"))
    val map /*             */ = bagSymbol.info.decl(TermName("map"))
    val flatMap /*         */ = bagSymbol.info.decl(TermName("flatMap"))
    val withFilter /*      */ = bagSymbol.info.decl(TermName("withFilter"))
    val groupBy /*         */ = bagSymbol.info.decl(TermName("groupBy"))
    val minus /*           */ = bagSymbol.info.decl(TermName("minus"))
    val plus /*            */ = bagSymbol.info.decl(TermName("plus"))
    val distinct /*        */ = bagSymbol.info.decl(TermName("distinct"))
    val fetchToStateless /**/ = statefulSymbol.info.decl(TermName("bag"))
    val updateWithZero /*  */ = statefulSymbol.info.decl(TermName("updateWithZero"))
    val updateWithOne /*   */ = statefulSymbol.info.decl(TermName("updateWithOne"))
    val updateWithMany /*  */ = statefulSymbol.info.decl(TermName("updateWithMany"))

    val methods = Set(
      read, write,
      stateful, fetchToStateless, updateWithZero, updateWithOne, updateWithMany,
      fold,
      map, flatMap, withFilter,
      groupBy,
      minus, plus, distinct
    ) ++ apply.alternatives

    val monadic = Set(map, flatMap, withFilter)
    val updateWith = Set(updateWithZero, updateWithOne, updateWithMany)

    // Type constructors
    val DATA_BAG /*        */ = typeOf[eu.stratosphere.emma.api.DataBag[Nothing]].typeConstructor
    val GROUP /*           */ = typeOf[eu.stratosphere.emma.api.Group[Nothing, Nothing]].typeConstructor
  }

  protected[emma] object IR {
    val module /*          */ = rootMirror.staticModule("eu.stratosphere.emma.compiler.ir.package").asModule

    val flatten /*         */ = module.info.decl(TermName("flatten")).asMethod
    val generator /*       */ = module.info.decl(TermName("generator")).asMethod
    val comprehension /*   */ = module.info.decl(TermName("comprehension")).asMethod
    val guard /*           */ = module.info.decl(TermName("guard")).asMethod
    val head /*            */ = module.info.decl(TermName("head")).asMethod

    val comprehensionOps /**/ = Set(flatten, generator, comprehension, guard, head)
  }

  /** Common validation helpers. */
  object Validation {

    val ok = ()
    val pass = Good(ok)<commit_msg>[IR] Re-formatted `Common.API` and `Common.IR`.<commit_after>package eu.stratosphere.emma
package compiler

import ast.AST

import org.scalactic._
import org.scalactic.Accumulation._

/** Common IR tools. */
trait Common extends ReflectUtil with AST {

  import universe._

  // --------------------------------------------------------------------------
  // Emma API
  // --------------------------------------------------------------------------

  /** A set of API method symbols to be comprehended. */
  protected[emma] object API {
    //@formatter:off
    val moduleSymbol          = rootMirror.staticModule("eu.stratosphere.emma.api.package")
    val bagSymbol             = rootMirror.staticClass("eu.stratosphere.emma.api.DataBag")
    val groupSymbol           = rootMirror.staticClass("eu.stratosphere.emma.api.Group")
    val statefulSymbol        = rootMirror.staticClass("eu.stratosphere.emma.api.Stateful.Bag")
    val inputFmtSymbol        = rootMirror.staticClass("eu.stratosphere.emma.api.InputFormat")
    val outputFmtSymbol       = rootMirror.staticClass("eu.stratosphere.emma.api.OutputFormat")

    val apply                 = bagSymbol.companion.info.decl(TermName("apply"))
    val read                  = moduleSymbol.info.decl(TermName("read"))
    val write                 = moduleSymbol.info.decl(TermName("write"))
    val stateful              = moduleSymbol.info.decl(TermName("stateful"))
    val fold                  = bagSymbol.info.decl(TermName("fold"))
    val map                   = bagSymbol.info.decl(TermName("map"))
    val flatMap               = bagSymbol.info.decl(TermName("flatMap"))
    val withFilter            = bagSymbol.info.decl(TermName("withFilter"))
    val groupBy               = bagSymbol.info.decl(TermName("groupBy"))
    val minus                 = bagSymbol.info.decl(TermName("minus"))
    val plus                  = bagSymbol.info.decl(TermName("plus"))
    val distinct              = bagSymbol.info.decl(TermName("distinct"))
    val fetchToStateless      = statefulSymbol.info.decl(TermName("bag"))
    val updateWithZero        = statefulSymbol.info.decl(TermName("updateWithZero"))
    val updateWithOne         = statefulSymbol.info.decl(TermName("updateWithOne"))
    val updateWithMany        = statefulSymbol.info.decl(TermName("updateWithMany"))

    val methods = Set(
      read, write,
      stateful, fetchToStateless, updateWithZero, updateWithOne, updateWithMany,
      fold,
      map, flatMap, withFilter,
      groupBy,
      minus, plus, distinct
    ) ++ apply.alternatives

    val monadic = Set(map, flatMap, withFilter)
    val updateWith = Set(updateWithZero, updateWithOne, updateWithMany)

    // Type constructors
    val DATA_BAG              = typeOf[eu.stratosphere.emma.api.DataBag[Nothing]].typeConstructor
    val GROUP                 = typeOf[eu.stratosphere.emma.api.Group[Nothing, Nothing]].typeConstructor
    //@formatter:on
  }

  protected[emma] object IR {
    //@formatter:off
    val module                = rootMirror.staticModule("eu.stratosphere.emma.compiler.ir.ComprehensionSyntax").asModule

    val flatten               = module.info.decl(TermName("flatten")).asMethod
    val generator             = module.info.decl(TermName("generator")).asMethod
    val comprehension         = module.info.decl(TermName("comprehension")).asMethod
    val guard                 = module.info.decl(TermName("guard")).asMethod
    val head                  = module.info.decl(TermName("head")).asMethod

    val comprehensionOps      = Set(flatten, generator, comprehension, guard, head)
    //@formatter:on
  }

  /** Common validation helpers. */
  object Validation {

    val ok = ()
    val pass = Good(ok)
{'fraction_non_alphanumeric': 0.1649343544857768, 'fraction_numerical': 0.0, 'mean_word_length': 2.3469107551487416, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3524298', 'n_tokens_mistral': 2105, 'n_tokens_neox': 2029, 'n_words': 361}