<commit_before>package dotty.tools
package dotc
package transform

import dotty.tools.dotc.ast.tpd
import dotty.tools.dotc.core.Annotations.ConcreteAnnotation
import dotty.tools.dotc.core.Contexts.Context
import dotty.tools.dotc.core.DenotTransformers.{InfoTransformer, DenotTransformer}
import dotty.tools.dotc.core.Denotations.SingleDenotation
import dotty.tools.dotc.core.Phases.Phase
import dotty.tools.dotc.core.SymDenotations.SymDenotation
import dotty.tools.dotc.core.Symbols.Symbol
import dotty.tools.dotc.core.Flags.PackageVal
import dotty.tools.dotc.core.Mode
import dotty.tools.dotc.ast.Trees._
import dotty.tools.dotc.core.Decorators._
import dotty.tools.dotc.util.DotClass
import scala.annotation.tailrec
import config.Printers.transforms
import scala.util.control.NonFatal

object TreeTransforms {
  import tpd._

  /** The base class of tree transforms. For each kind of tree K, there are
   *  two methods which can be overridden:
   *
   *  prepareForK // return a new TreeTransform which gets applied to the K
   *  // node and its children
   *  transformK // transform node of type K
   *
   *  If a transform does not need to visit a node or any of its children, it
   *  signals this fact by returning a NoTransform from a prepare method.
   *
   *  If all transforms in a group are NoTransforms, the tree is no longer traversed.
   *
   *
   *            Performance analysis: Taking the dotty compiler frontend as a use case, we are aiming for a warm performance of
   *            about 4000 lines / sec. This means 6 seconds for a codebase of 24'000 lines. Of these the frontend consumes
   *            over 2.5 seconds, erasure and code generation will most likely consume over 1 second each. So we would have
   *            about 1 sec for all other transformations in our budget. Of this second, let's assume a maximum of 20% for
   *            the general dispatch overhead as opposed to the concrete work done in transformations. So that leaves us with
   *            0.2sec, or roughly 600M processor cycles.
   *
   *            Now, to the amount of work that needs to be done. The codebase produces of about 250'000 trees after typechecking.
   *            Transformations are likely to make this bigger so let's assume 300K trees on average. We estimate to have about 100
   *            micro-transformations. Let's say 5 transformation groups of 20 micro-transformations each. (by comparison,
   *            scalac has in excess of 20 phases, and most phases do multiple transformations). There are then 30M visits
   *            of a node by a transformation. Each visit has a budget of 20 processor cycles.
   *
   *            A more detailed breakdown: I assume that about one third of all transformations have real work to do for each node.
   *            This might look high, but keep in mind that the most common nodes are Idents and Selects, and most transformations
   *            touch these. By contrast the amount of work for generating new transformations should be negligible.
   *
   *            So, in 400 clock cycles we need to (1) perform a pattern match according to the type of node, (2) generate new
   *            transformations if applicable, (3) reconstitute the tree node from the result of transforming the children, and
   *            (4) chain 7 out of 20 transformations over the resulting tree node. I believe the current algorithm is suitable
   *            for achieving this goal, but there can be no wasted cycles anywhere.
   */
  abstract class TreeTransform extends DotClass {

    def phase: MiniPhase

    def treeTransformPhase: Phase = phase.next

    def prepareForIdent(tree: Ident)(implicit ctx: Context) = this
    def prepareForSelect(tree: Select)(implicit ctx: Context) = this
    def prepareForThis(tree: This)(implicit ctx: Context) = this
    def prepareForSuper(tree: Super)(implicit ctx: Context) = this
    def prepareForApply(tree: Apply)(implicit ctx: Context) = this
    def prepareForTypeApply(tree: TypeApply)(implicit ctx: Context) = this
    def prepareForLiteral(tree: Literal)(implicit ctx: Context) = this
    def prepareForPair(tree: Pair)(implicit ctx: Context) = this
    def prepareForNew(tree: New)(implicit ctx: Context) = this
    def prepareForTyped(tree: Typed)(implicit ctx: Context) = this
    def prepareForAssign(tree: Assign)(implicit ctx: Context) = this
    def prepareForBlock(tree: Block)(implicit ctx: Context) = this
    def prepareForIf(tree: If)(implicit ctx: Context) = this
    def prepareForClosure(tree: Closure)(implicit ctx: Context) = this
    def prepareForMatch(tree: Match)(implicit ctx: Context) = this
    def prepareForCaseDef(tree: CaseDef)(implicit ctx: Context) = this
    def prepareForReturn(tree: Return)(implicit ctx: Context) = this
    def prepareForTry(tree: Try)(implicit ctx: Context) = this
    def prepareForSeqLiteral(tree: SeqLiteral)(implicit ctx: Context) = this
    def prepareForTypeTree(tree: TypeTree)(implicit ctx: Context) = this
    def prepareForSelectFromTypeTree(tree: SelectFromTypeTree)(implicit ctx: Context) = this
    def prepareForBind(tree: Bind)(implicit ctx: Context) = this
    def prepareForAlternative(tree: Alternative)(implicit ctx: Context) = this
    def prepareForTypeDef(tree: TypeDef)(implicit ctx: Context) = this
    def prepareForUnApply(tree: UnApply)(implicit ctx: Context) = this
    def prepareForValDef(tree: ValDef)(implicit ctx: Context) = this
    def prepareForDefDef(tree: DefDef)(implicit ctx: Context) = this
    def prepareForTemplate(tree: Template)(implicit ctx: Context) = this
    def prepareForPackageDef(tree: PackageDef)(implicit ctx: Context) = this
    def prepareForStats(trees: List[Tree])(implicit ctx: Context) = this

    def prepareForUnit(tree: Tree)(implicit ctx: Context) = this

    def transformIdent(tree: Ident)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformSelect(tree: Select)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformThis(tree: This)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformSuper(tree: Super)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformApply(tree: Apply)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformTypeApply(tree: TypeApply)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformLiteral(tree: Literal)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformNew(tree: New)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformPair(tree: Pair)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformTyped(tree: Typed)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformAssign(tree: Assign)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformBlock(tree: Block)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformIf(tree: If)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformClosure(tree: Closure)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformMatch(tree: Match)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformCaseDef(tree: CaseDef)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformReturn(tree: Return)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformTry(tree: Try)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformSeqLiteral(tree: SeqLiteral)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformTypeTree(tree: TypeTree)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformSelectFromTypeTree(tree: SelectFromTypeTree)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformBind(tree: Bind)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformAlternative(tree: Alternative)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformUnApply(tree: UnApply)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformValDef(tree: ValDef)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformDefDef(tree: DefDef)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformTypeDef(tree: TypeDef)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformTemplate(tree: Template)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformPackageDef(tree: PackageDef)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformStats(trees: List[Tree])(implicit ctx: Context, info: TransformerInfo): List[Tree] = trees
    def transformOther(tree: Tree)(implicit ctx: Context, info: TransformerInfo): Tree = tree

    def transformUnit(tree: Tree)(implicit ctx: Context, info: TransformerInfo): Tree = tree

    /** Transform tree using all transforms of current group (including this one) */
    def transform(tree: Tree)(implicit ctx: Context, info: TransformerInfo): Tree = info.group.transform(tree, info, 0)

    /** Transform subtree using all transforms following the current one in this group */
    def transformFollowingDeep(tree: Tree)(implicit ctx: Context, info: TransformerInfo): Tree = info.group.transform(tree, info, phase.idx + 1)

    /** Transform single node using all transforms following the current one in this group */
    def transformFollowing(tree: Tree)(implicit ctx: Context, info: TransformerInfo): Tree = info.group.transformSingle(tree, phase.idx + 1)

    def atGroupEnd[T](action : Context => T)(implicit ctx: Context, info: TransformerInfo) = {
      val last = info.transformers(info.transformers.length - 1)
      action(ctx.withPhase(last.phase.next))
    }
  }

  /** A phase that defines a TreeTransform to be used in a group */
  trait MiniPhase extends Phase { thisPhase =>
    def treeTransform: TreeTransform

    /** id of this mini phase in group */
    var idx: Int = _

    /** List of names of phases that should have finished their processing of all compilation units
     *  before this phase starts
     */
    def runsAfterGroupsOf: Set[Class[_ <: Phase]] = Set.empty

    protected def mkTreeTransformer = new TreeTransformer {
      override def phaseName: String = thisPhase.phaseName
      override def miniPhases = Array(thisPhase)
    }

    override def run(implicit ctx: Context): Unit = {
      mkTreeTransformer.run
    }
  }

  /** A mini phase that is its own tree transform */
  abstract class MiniPhaseTransform extends TreeTransform with MiniPhase {
    def treeTransform = this
    def phase = this
 }

  /** A helper trait to transform annotations on MemberDefs */
  trait AnnotationTransformer extends MiniPhaseTransform with InfoTransformer {

    val annotationTransformer = mkTreeTransformer
    override final def treeTransformPhase = this
      // need to run at own phase because otherwise we get ahead of ourselves in transforming denotations

    override def transform(ref: SingleDenotation)(implicit ctx: Context): SingleDenotation = {
      val info1 = transformInfo(ref.info, ref.symbol)

      ref match {
        case ref: SymDenotation =>
          val annotTrees = ref.annotations.map(_.tree)
          val annotTrees1 = annotTrees.mapConserve(annotationTransformer.macroTransform)
          val annots1 = if (annotTrees eq annotTrees1) ref.annotations else annotTrees1.map(new ConcreteAnnotation(_))
          if ((info1 eq ref.info) && (annots1 eq ref.annotations)) ref
          else ref.copySymDenotation(info = info1, annotations = annots1)
        case _ => if (info1 eq ref.info) ref else ref.derivedSingleDenotation(ref.symbol, info1)
      }
    }
  }

  @sharable val NoTransform = new TreeTransform {
    def phase = unsupported("phase")
  }

  type Mutator[T] = (TreeTransform, T, Context) => TreeTransform

  class TransformerInfo(val transformers: Array[TreeTransform], val nx: NXTransformations, val group: TreeTransformer)

  /** This class maintains track of which methods are redefined in MiniPhases and creates execution plans for transformXXX and prepareXXX
   *  Thanks to Martin for this idea
   *  @see NXTransformations.index for format of plan
   */
  class NXTransformations {

    private def hasRedefinedMethod(cls: Class[_], name: String): Boolean =<commit_msg>Transform annotations only if defined in current run<commit_after>package dotty.tools
package dotc
package transform

import dotty.tools.dotc.ast.tpd
import dotty.tools.dotc.core.Annotations.ConcreteAnnotation
import dotty.tools.dotc.core.Contexts.Context
import dotty.tools.dotc.core.DenotTransformers.{InfoTransformer, DenotTransformer}
import dotty.tools.dotc.core.Denotations.SingleDenotation
import dotty.tools.dotc.core.Phases.Phase
import dotty.tools.dotc.core.SymDenotations.SymDenotation
import dotty.tools.dotc.core.Symbols.Symbol
import dotty.tools.dotc.core.Flags.PackageVal
import dotty.tools.dotc.core.Mode
import dotty.tools.dotc.ast.Trees._
import dotty.tools.dotc.core.Decorators._
import dotty.tools.dotc.util.DotClass
import scala.annotation.tailrec
import config.Printers.transforms
import scala.util.control.NonFatal

object TreeTransforms {
  import tpd._

  /** The base class of tree transforms. For each kind of tree K, there are
   *  two methods which can be overridden:
   *
   *  prepareForK // return a new TreeTransform which gets applied to the K
   *  // node and its children
   *  transformK // transform node of type K
   *
   *  If a transform does not need to visit a node or any of its children, it
   *  signals this fact by returning a NoTransform from a prepare method.
   *
   *  If all transforms in a group are NoTransforms, the tree is no longer traversed.
   *
   *
   *            Performance analysis: Taking the dotty compiler frontend as a use case, we are aiming for a warm performance of
   *            about 4000 lines / sec. This means 6 seconds for a codebase of 24'000 lines. Of these the frontend consumes
   *            over 2.5 seconds, erasure and code generation will most likely consume over 1 second each. So we would have
   *            about 1 sec for all other transformations in our budget. Of this second, let's assume a maximum of 20% for
   *            the general dispatch overhead as opposed to the concrete work done in transformations. So that leaves us with
   *            0.2sec, or roughly 600M processor cycles.
   *
   *            Now, to the amount of work that needs to be done. The codebase produces of about 250'000 trees after typechecking.
   *            Transformations are likely to make this bigger so let's assume 300K trees on average. We estimate to have about 100
   *            micro-transformations. Let's say 5 transformation groups of 20 micro-transformations each. (by comparison,
   *            scalac has in excess of 20 phases, and most phases do multiple transformations). There are then 30M visits
   *            of a node by a transformation. Each visit has a budget of 20 processor cycles.
   *
   *            A more detailed breakdown: I assume that about one third of all transformations have real work to do for each node.
   *            This might look high, but keep in mind that the most common nodes are Idents and Selects, and most transformations
   *            touch these. By contrast the amount of work for generating new transformations should be negligible.
   *
   *            So, in 400 clock cycles we need to (1) perform a pattern match according to the type of node, (2) generate new
   *            transformations if applicable, (3) reconstitute the tree node from the result of transforming the children, and
   *            (4) chain 7 out of 20 transformations over the resulting tree node. I believe the current algorithm is suitable
   *            for achieving this goal, but there can be no wasted cycles anywhere.
   */
  abstract class TreeTransform extends DotClass {

    def phase: MiniPhase

    def treeTransformPhase: Phase = phase.next

    def prepareForIdent(tree: Ident)(implicit ctx: Context) = this
    def prepareForSelect(tree: Select)(implicit ctx: Context) = this
    def prepareForThis(tree: This)(implicit ctx: Context) = this
    def prepareForSuper(tree: Super)(implicit ctx: Context) = this
    def prepareForApply(tree: Apply)(implicit ctx: Context) = this
    def prepareForTypeApply(tree: TypeApply)(implicit ctx: Context) = this
    def prepareForLiteral(tree: Literal)(implicit ctx: Context) = this
    def prepareForPair(tree: Pair)(implicit ctx: Context) = this
    def prepareForNew(tree: New)(implicit ctx: Context) = this
    def prepareForTyped(tree: Typed)(implicit ctx: Context) = this
    def prepareForAssign(tree: Assign)(implicit ctx: Context) = this
    def prepareForBlock(tree: Block)(implicit ctx: Context) = this
    def prepareForIf(tree: If)(implicit ctx: Context) = this
    def prepareForClosure(tree: Closure)(implicit ctx: Context) = this
    def prepareForMatch(tree: Match)(implicit ctx: Context) = this
    def prepareForCaseDef(tree: CaseDef)(implicit ctx: Context) = this
    def prepareForReturn(tree: Return)(implicit ctx: Context) = this
    def prepareForTry(tree: Try)(implicit ctx: Context) = this
    def prepareForSeqLiteral(tree: SeqLiteral)(implicit ctx: Context) = this
    def prepareForTypeTree(tree: TypeTree)(implicit ctx: Context) = this
    def prepareForSelectFromTypeTree(tree: SelectFromTypeTree)(implicit ctx: Context) = this
    def prepareForBind(tree: Bind)(implicit ctx: Context) = this
    def prepareForAlternative(tree: Alternative)(implicit ctx: Context) = this
    def prepareForTypeDef(tree: TypeDef)(implicit ctx: Context) = this
    def prepareForUnApply(tree: UnApply)(implicit ctx: Context) = this
    def prepareForValDef(tree: ValDef)(implicit ctx: Context) = this
    def prepareForDefDef(tree: DefDef)(implicit ctx: Context) = this
    def prepareForTemplate(tree: Template)(implicit ctx: Context) = this
    def prepareForPackageDef(tree: PackageDef)(implicit ctx: Context) = this
    def prepareForStats(trees: List[Tree])(implicit ctx: Context) = this

    def prepareForUnit(tree: Tree)(implicit ctx: Context) = this

    def transformIdent(tree: Ident)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformSelect(tree: Select)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformThis(tree: This)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformSuper(tree: Super)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformApply(tree: Apply)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformTypeApply(tree: TypeApply)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformLiteral(tree: Literal)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformNew(tree: New)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformPair(tree: Pair)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformTyped(tree: Typed)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformAssign(tree: Assign)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformBlock(tree: Block)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformIf(tree: If)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformClosure(tree: Closure)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformMatch(tree: Match)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformCaseDef(tree: CaseDef)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformReturn(tree: Return)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformTry(tree: Try)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformSeqLiteral(tree: SeqLiteral)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformTypeTree(tree: TypeTree)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformSelectFromTypeTree(tree: SelectFromTypeTree)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformBind(tree: Bind)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformAlternative(tree: Alternative)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformUnApply(tree: UnApply)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformValDef(tree: ValDef)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformDefDef(tree: DefDef)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformTypeDef(tree: TypeDef)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformTemplate(tree: Template)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformPackageDef(tree: PackageDef)(implicit ctx: Context, info: TransformerInfo): Tree = tree
    def transformStats(trees: List[Tree])(implicit ctx: Context, info: TransformerInfo): List[Tree] = trees
    def transformOther(tree: Tree)(implicit ctx: Context, info: TransformerInfo): Tree = tree

    def transformUnit(tree: Tree)(implicit ctx: Context, info: TransformerInfo): Tree = tree

    /** Transform tree using all transforms of current group (including this one) */
    def transform(tree: Tree)(implicit ctx: Context, info: TransformerInfo): Tree = info.group.transform(tree, info, 0)

    /** Transform subtree using all transforms following the current one in this group */
    def transformFollowingDeep(tree: Tree)(implicit ctx: Context, info: TransformerInfo): Tree = info.group.transform(tree, info, phase.idx + 1)

    /** Transform single node using all transforms following the current one in this group */
    def transformFollowing(tree: Tree)(implicit ctx: Context, info: TransformerInfo): Tree = info.group.transformSingle(tree, phase.idx + 1)

    def atGroupEnd[T](action : Context => T)(implicit ctx: Context, info: TransformerInfo) = {
      val last = info.transformers(info.transformers.length - 1)
      action(ctx.withPhase(last.phase.next))
    }
  }

  /** A phase that defines a TreeTransform to be used in a group */
  trait MiniPhase extends Phase { thisPhase =>
    def treeTransform: TreeTransform

    /** id of this mini phase in group */
    var idx: Int = _

    /** List of names of phases that should have finished their processing of all compilation units
     *  before this phase starts
     */
    def runsAfterGroupsOf: Set[Class[_ <: Phase]] = Set.empty

    protected def mkTreeTransformer = new TreeTransformer {
      override def phaseName: String = thisPhase.phaseName
      override def miniPhases = Array(thisPhase)
    }

    override def run(implicit ctx: Context): Unit = {
      mkTreeTransformer.run
    }
  }

  /** A mini phase that is its own tree transform */
  abstract class MiniPhaseTransform extends TreeTransform with MiniPhase {
    def treeTransform = this
    def phase = this
 }

  /** A helper trait to transform annotations on MemberDefs */
  trait AnnotationTransformer extends MiniPhaseTransform with InfoTransformer {

    val annotationTransformer = mkTreeTransformer
    override final def treeTransformPhase = this
      // need to run at own phase because otherwise we get ahead of ourselves in transforming denotations

    override def transform(ref: SingleDenotation)(implicit ctx: Context): SingleDenotation = {
      val info1 = transformInfo(ref.info, ref.symbol)

      ref match {
        case ref: SymDenotation =>
          val annots1 =
            if (!ref.symbol.isDefinedInCurrentRun) ref.annotations // leave annotations read from class files alone
            else {
              val annotTrees = ref.annotations.map(_.tree)
              val annotTrees1 = annotTrees.mapConserve(annotationTransformer.macroTransform)
              if (annotTrees eq annotTrees1) ref.annotations
              else annotTrees1.map(new ConcreteAnnotation(_))
            }
          if ((info1 eq ref.info) && (annots1 eq ref.annotations)) ref
          else ref.copySymDenotation(info = info1, annotations = annots1)
        case _ =>
          if (info1 eq ref.info) ref
          else ref.derivedSingleDenotation(ref.symbol, info1)
      }
    }
  }

  @sharable val NoTransform = new TreeTransform {
    def phase = unsupported("phase")
  }

  type Mutator[T] = (TreeTransform, T, Context) => TreeTransform

  class TransformerInfo(val transformers: Array[TreeTransform], val nx: NXTransformations, val group: TreeTransformer)

  /** This class maintains track of which methods are redefined in MiniPhases and creates execution plans for transformXXX and prepareXXX
   *  Thanks to Martin for this idea
   *  @see NXTransformations.index for format of plan
   */
  class NXTransformations {

    private def hasRedefinedMethod(cls: Class[_], name: String): Boolean =
{'fraction_non_alphanumeric': 0.08114139964929061, 'fraction_numerical': 0.00534034752112227, 'mean_word_length': 3.944433497536946, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1080176', 'n_tokens_mistral': 6917, 'n_tokens_neox': 6660, 'n_words': 2534}