<commit_before>/**
  * Copyright (C) 2017 Orbeon, Inc.
  *
  * This program is free software; you can redistribute it and/or modify it under the terms of the
  * GNU Lesser General Public License as published by the Free Software Foundation; either version
  *  2.1 of the License, or (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
  * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  * See the GNU Lesser General Public License for more details.
  *
  * The full text of the license is available at http://www.gnu.org/copyleft/lesser.html
  */
package org.orbeon.oxf.fb

import org.orbeon.builder.rpc.FormBuilderRpcApiImpl
import org.orbeon.datatypes.{AboveBelow, Direction, MediatypeRange}
import org.orbeon.oxf.fb.FormBuilder._
import org.orbeon.oxf.fb.Undo.UndoOrRedo
import org.orbeon.oxf.fb.UndoAction._
import org.orbeon.oxf.fr
import org.orbeon.oxf.fr.FormRunner.findControlByName
import org.orbeon.oxf.fr.NodeInfoCell._
import org.orbeon.oxf.fr.{FormRunner, Names}
import org.orbeon.oxf.util.CoreUtils._
import org.orbeon.oxf.util.StringUtils._
import org.orbeon.oxf.xforms.action.XFormsAPI._
import org.orbeon.oxf.xforms.analysis.controls.LHHA
import org.orbeon.oxf.xforms.analysis.model.Model
import org.orbeon.oxf.xforms.control.XFormsSingleNodeControl
import org.orbeon.oxf.xml.{SaxonUtils, TransformerUtils}
import org.orbeon.saxon.ArrayFunctions
import org.orbeon.saxon.function.Property
import org.orbeon.saxon.om.{NodeInfo, SequenceIterator}
import org.orbeon.scaxon.Implicits._
import org.orbeon.scaxon.NodeConversions._
import org.orbeon.scaxon.SimplePath._
import org.orbeon.xforms.XFormsId

object FormBuilderXPathApi {

  // This is called when the user adds/removes an iteration, as we want to update the templates in this case in order
  // to adjust the default number of iterations. See https://github.com/orbeon/orbeon-forms/issues/2379
  //@XPathFunction
  def updateTemplatesFromDynamicIterationChange(controlName: String): Unit = {

    implicit val ctx = FormBuilderDocContext()

    FormRunner.findControlByName(ctx.formDefinitionRootElem, FormRunner.controlNameFromId(controlName)) foreach { controlElem ⇒
      assert(FormRunner.isRepeat(controlElem))
      updateTemplatesCheckContainers(FormRunner.findAncestorRepeatNames(controlElem).to[Set])
    }
  }

  // Rename a control's nested iteration if any
  //@XPathFunction
  def renameControlIterationIfNeeded(
    oldControlName             : String,
    newControlName             : String,
    oldChildElementNameOrBlank : String,
    newChildElementNameOrBlank : String
  ): Unit =
    FormBuilder.renameControlIterationIfNeeded(
      oldControlName,
      newControlName,
      oldChildElementNameOrBlank.trimAllToOpt,
      newChildElementNameOrBlank.trimAllToOpt
    )(FormBuilderDocContext())

  //@XPathFunction
  def renameControlIfNeeded(oldName: String, newName: String, addToUndoStack: Boolean): Unit = {
    implicit val ctx = FormBuilderDocContext()
    FormBuilder.renameControlIfNeeded(oldName, newName) filter
      (_ ⇒ addToUndoStack) foreach
      Undo.pushUserUndoAction
  }

  // Get the normalized value of a computed MIP for the given control name and computed MIP name
  //@XPathFunction
  def readDenormalizedCalculatedMip(controlName: String, mipName: String): String = {

    implicit val ctx = FormBuilderDocContext()

    val resultOpt =
      for {
        mip      ← Model.AllComputedMipsByName.get(mipName)
        bindElem ← FormRunner.findBindByName(ctx.formDefinitionRootElem, controlName)
      } yield
        FormBuilder.readDenormalizedCalculatedMip(bindElem, mip)

    resultOpt getOrElse (throw new IllegalArgumentException)
  }

  //@XPathFunction
  def writeAndNormalizeCalculatedMip(controlName: String, mipName: String, mipValue: String): Unit = {

    implicit val ctx = FormBuilderDocContext()

    val resultOpt =
      for {
        mip ← Model.AllComputedMipsByName.get(mipName)
      } yield
        FormBuilder.writeAndNormalizeMip(controlName, mip, mipValue)

    resultOpt getOrElse (throw new IllegalArgumentException)
  }

  //@XPathFunction
  def setRepeatProperties(
    controlName          : String,
    repeat               : Boolean,
    userCanAddRemove     : Boolean,
    usePaging            : Boolean,
    min                  : String,
    max                  : String,
    freeze               : String,
    iterationNameOrEmpty : String,
    applyDefaults        : Boolean,
    initialIterations    : String
  ): Unit =
    FormBuilder.setRepeatProperties(
      controlName          = controlName,
      repeat               = repeat,
      userCanAddRemove     = userCanAddRemove,
      usePaging            = usePaging,
      min                  = min,
      max                  = max,
      freeze               = freeze,
      iterationNameOrEmpty = iterationNameOrEmpty,
      applyDefaults        = applyDefaults,
      initialIterations    = initialIterations
    )(FormBuilderDocContext())

  //@XPathFunction
  def selectCellForControlId(controlId: String): Unit = {
    implicit val ctx = FormBuilderDocContext()
    FormRunner.findControlByName(ctx.formDefinitionRootElem, FormRunner.controlNameFromId(controlId)).to[List] flatMap
      (_ parent CellTest) foreach selectCell
  }

    // Write back everything
  //@XPathFunction
  def writeAlertsAndValidationsAsXML(
    controlName      : String,
    newAppearance    : String,
    defaultAlertElem : NodeInfo,
    validationElems  : Array[NodeInfo]
  ): Unit =
    FormBuilder.writeAlertsAndValidationsAsXML(
      controlName,
      newAppearance,
      defaultAlertElem,
      validationElems
    )(FormBuilderDocContext())

  //@XPathFunction
  def setControlLHHAMediatype(controlName: String, lhha: String, isHTML: Boolean): Unit =
    FormBuilder.setControlLhhatMediatype(controlName, lhha, isHTML)(FormBuilderDocContext())

  //@XPathFunction
  def setControlLabelHintHelpOrText(
    controlName : String,
    lhht        : String,
    value       : String,
    params      : Array[NodeInfo],
    isHTML      : Boolean
  ): Unit = {

    implicit val ctx = FormBuilderDocContext()

    val isOptionalLHHAT =
      lhht == LHHA.Help.entryName || lhht == fr.XMLNames.FRIterationLabelQName.localName

    // Make sure an optional element is present while we set content or attributes
    if (isOptionalLHHAT)
      FormBuilder.ensureCleanLHHAElements(controlName, lhht, count = 1, replace = true)

    FormBuilder.setControlLabelHintHelpOrText(controlName, lhht, value, Some(params), isHTML)

    // If an optional element turns out to be empty, then remove it
    if (isOptionalLHHAT) {

      val (doDelete, holders) =
        holdersToRemoveIfHasBlankOrMissingLHHAForAllLangs(controlName, FormBuilder.getControlLhhat(controlName, lhht).toList, lhht)

      if (doDelete) {
        delete(holders)
        delete(findControlByName(ctx.formDefinitionRootElem, controlName).toList child controlLHHATQName(lhht))
      }
    }
  }

  // Set the control's items for all languages
  //@XPathFunction
  def setControlItems(controlName: String, items: NodeInfo): Unit =
    FormBuilder.setControlItems(controlName, items)(FormBuilderDocContext())

  // For a given control, set the mediatype on the itemset labels to be HTML or plain text
  //@XPathFunction
  def setItemsetHTMLMediatype(controlName: String, isHTML: Boolean): Unit = {

    implicit val ctx = FormBuilderDocContext()

    if (isHTML != FormBuilder.isItemsetHTMLMediatype(controlName)) {
      val itemsetEl = FormRunner.findControlByName(ctx.formDefinitionRootElem, controlName).toList child "itemset"
      val labelHintEls = Seq("label", "hint") flatMap (itemsetEl.child(_))
      setHTMLMediatype(labelHintEls, isHTML)
    }
  }

  //@XPathFunction
  def isItemsetHTMLMediatype(controlName: String): Boolean =
    FormBuilder.isItemsetHTMLMediatype(controlName)(FormBuilderDocContext())

  //@XPathFunction
  def initializeGrids(doc: NodeInfo): Unit = {

    // Create context explicitly based on the document passed, as the node might not be
    // in the main Form Builder instance yet.
    implicit val ctx = FormBuilderDocContext(doc)

    // 1. Annotate all the grid and grid cells of the given document with unique ids,
    // if they don't have them already. We do this so that ids are stable as we move
    // things around, otherwise if the XForms document is recreated new automatic ids
    // are generated for objects without id.
    FormBuilder.annotateGridsAndCells(ctx.bodyElem)

    // 2. Select the first td if any
    ctx.bodyElem descendant GridTest descendant CellTest take 1 foreach selectCell
  }

  //@XPathFunction
  def findModelElem(doc: NodeInfo): NodeInfo =
    FormBuilderDocContext(doc).modelElem

  //@XPathFunction
  def findDataInstanceElem(doc: NodeInfo): NodeInfo =
    FormBuilderDocContext(doc).dataInstanceElem

  //@XPathFunction
  def findMetadataInstanceElem(doc: NodeInfo): NodeInfo =
    FormBuilderDocContext(doc).metadataInstanceElem

  // Create template content from a bind name
  // FIXME: Saxon can pass null as `bindings`.
  //@XPathFunction
  def createTemplateContentFromBindName(inDoc: NodeInfo, name: String, bindings: List[NodeInfo]): Option[NodeInfo] =
    FormBuilder.createTemplateContentFromBindName(name, Option(bindings) getOrElse Nil)(FormBuilderDocContext(inDoc))

  // See: https://github.com/orbeon/orbeon-forms/issues/633
  // See: https://github.com/orbeon/orbeon-forms/issues/3073
  //@XPathFunction
  def updateSectionTemplateContentHolders(inDoc: NodeInfo): Unit = {

    // Create context explicitly based on the document passed, as the node might not be
    // in the main Form Builder instance yet.
    implicit val ctx = FormBuilderDocContext(inDoc)

    // Find data holders for all section templates
    val holdersWithRoots =
      for {
        sectionNode   ← FormRunner.findSectionsWithTemplates(ctx.bodyElem)
        controlName   ← FormRunner.getControlNameOpt(sectionNode).toList
        holder        ← FormBuilder.findDataHolders(controlName) // TODO: What about within repeated sections? Templates ok?
        componentNode ← FormRunner.findComponentNodeForSection(sectionNode)
        xblNode       ← FormRunner.findXblXblForSectionTemplateNamespace(ctx.bodyElem, componentNode.namespaceURI)
        bindingNode   ← FormRunner.findXblBindingForLocalname(xblNode, componentNode.localname)
        instance      ← FormRunner.findXblInstance(bindingNode, fr.Names.FormTemplate)
        instanceRoot  ← instance / * headOption
      } yield
        holder → instanceRoot

    holdersWithRoots foreach { case (holder, instanceRoot) ⇒
      delete(holder / *)
      insert(into = holder, origin = instanceRoot / *)
    }
  }

  // Find all resource holders and elements which are unneeded because the resources are blank
  //@XPathFunction
  def findBlankHelpHoldersAndElements(inDoc: NodeInfo): Seq[NodeInfo] = {

    // Create context explicitly based on the document passed, as the node might not be
    // in the main Form Builder instance yet.
    implicit val ctx = FormBuilderDocContext(inDoc)

    val lhha = LHHA.Help

    val lhhaTest: Test = LHHA.QNameForValue(lhha)

    val allHelpElementsWithControlNames =
      ctx.bodyElem descendant lhhaTest map
      (lhhaElem ⇒ lhhaElem → lhhaElem.attValue("ref")) collect
      { case (lhhaElem, HelpRefMatcher(controlName)) ⇒ lhhaElem → controlName }

    val allUnneededHolders =
      allHelpElementsWithControlNames flatMap { case (lhhaElement, controlName) ⇒
        holdersToRemoveIfHasBlankOrMissingLHHAForAllLangs(controlName, List(lhhaElement), lhha.entryName)._2
      }

    allUnneededHolders.flatten<commit_msg>For #4268: also remove `<xf:help>` element<commit_after>/**
  * Copyright (C) 2017 Orbeon, Inc.
  *
  * This program is free software; you can redistribute it and/or modify it under the terms of the
  * GNU Lesser General Public License as published by the Free Software Foundation; either version
  *  2.1 of the License, or (at your option) any later version.
  *
  * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
  * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
  * See the GNU Lesser General Public License for more details.
  *
  * The full text of the license is available at http://www.gnu.org/copyleft/lesser.html
  */
package org.orbeon.oxf.fb

import org.orbeon.builder.rpc.FormBuilderRpcApiImpl
import org.orbeon.datatypes.{AboveBelow, Direction, MediatypeRange}
import org.orbeon.oxf.fb.FormBuilder._
import org.orbeon.oxf.fb.Undo.UndoOrRedo
import org.orbeon.oxf.fb.UndoAction._
import org.orbeon.oxf.fr
import org.orbeon.oxf.fr.FormRunner.findControlByName
import org.orbeon.oxf.fr.NodeInfoCell._
import org.orbeon.oxf.fr.{FormRunner, Names}
import org.orbeon.oxf.util.CoreUtils._
import org.orbeon.oxf.util.StringUtils._
import org.orbeon.oxf.xforms.action.XFormsAPI._
import org.orbeon.oxf.xforms.analysis.controls.LHHA
import org.orbeon.oxf.xforms.analysis.model.Model
import org.orbeon.oxf.xforms.control.XFormsSingleNodeControl
import org.orbeon.oxf.xml.{SaxonUtils, TransformerUtils}
import org.orbeon.saxon.ArrayFunctions
import org.orbeon.saxon.function.Property
import org.orbeon.saxon.om.{NodeInfo, SequenceIterator}
import org.orbeon.scaxon.Implicits._
import org.orbeon.scaxon.NodeConversions._
import org.orbeon.scaxon.SimplePath._
import org.orbeon.xforms.XFormsId

object FormBuilderXPathApi {

  // This is called when the user adds/removes an iteration, as we want to update the templates in this case in order
  // to adjust the default number of iterations. See https://github.com/orbeon/orbeon-forms/issues/2379
  //@XPathFunction
  def updateTemplatesFromDynamicIterationChange(controlName: String): Unit = {

    implicit val ctx = FormBuilderDocContext()

    FormRunner.findControlByName(ctx.formDefinitionRootElem, FormRunner.controlNameFromId(controlName)) foreach { controlElem ⇒
      assert(FormRunner.isRepeat(controlElem))
      updateTemplatesCheckContainers(FormRunner.findAncestorRepeatNames(controlElem).to[Set])
    }
  }

  // Rename a control's nested iteration if any
  //@XPathFunction
  def renameControlIterationIfNeeded(
    oldControlName             : String,
    newControlName             : String,
    oldChildElementNameOrBlank : String,
    newChildElementNameOrBlank : String
  ): Unit =
    FormBuilder.renameControlIterationIfNeeded(
      oldControlName,
      newControlName,
      oldChildElementNameOrBlank.trimAllToOpt,
      newChildElementNameOrBlank.trimAllToOpt
    )(FormBuilderDocContext())

  //@XPathFunction
  def renameControlIfNeeded(oldName: String, newName: String, addToUndoStack: Boolean): Unit = {
    implicit val ctx = FormBuilderDocContext()
    FormBuilder.renameControlIfNeeded(oldName, newName) filter
      (_ ⇒ addToUndoStack) foreach
      Undo.pushUserUndoAction
  }

  // Get the normalized value of a computed MIP for the given control name and computed MIP name
  //@XPathFunction
  def readDenormalizedCalculatedMip(controlName: String, mipName: String): String = {

    implicit val ctx = FormBuilderDocContext()

    val resultOpt =
      for {
        mip      ← Model.AllComputedMipsByName.get(mipName)
        bindElem ← FormRunner.findBindByName(ctx.formDefinitionRootElem, controlName)
      } yield
        FormBuilder.readDenormalizedCalculatedMip(bindElem, mip)

    resultOpt getOrElse (throw new IllegalArgumentException)
  }

  //@XPathFunction
  def writeAndNormalizeCalculatedMip(controlName: String, mipName: String, mipValue: String): Unit = {

    implicit val ctx = FormBuilderDocContext()

    val resultOpt =
      for {
        mip ← Model.AllComputedMipsByName.get(mipName)
      } yield
        FormBuilder.writeAndNormalizeMip(controlName, mip, mipValue)

    resultOpt getOrElse (throw new IllegalArgumentException)
  }

  //@XPathFunction
  def setRepeatProperties(
    controlName          : String,
    repeat               : Boolean,
    userCanAddRemove     : Boolean,
    usePaging            : Boolean,
    min                  : String,
    max                  : String,
    freeze               : String,
    iterationNameOrEmpty : String,
    applyDefaults        : Boolean,
    initialIterations    : String
  ): Unit =
    FormBuilder.setRepeatProperties(
      controlName          = controlName,
      repeat               = repeat,
      userCanAddRemove     = userCanAddRemove,
      usePaging            = usePaging,
      min                  = min,
      max                  = max,
      freeze               = freeze,
      iterationNameOrEmpty = iterationNameOrEmpty,
      applyDefaults        = applyDefaults,
      initialIterations    = initialIterations
    )(FormBuilderDocContext())

  //@XPathFunction
  def selectCellForControlId(controlId: String): Unit = {
    implicit val ctx = FormBuilderDocContext()
    FormRunner.findControlByName(ctx.formDefinitionRootElem, FormRunner.controlNameFromId(controlId)).to[List] flatMap
      (_ parent CellTest) foreach selectCell
  }

    // Write back everything
  //@XPathFunction
  def writeAlertsAndValidationsAsXML(
    controlName      : String,
    newAppearance    : String,
    defaultAlertElem : NodeInfo,
    validationElems  : Array[NodeInfo]
  ): Unit =
    FormBuilder.writeAlertsAndValidationsAsXML(
      controlName,
      newAppearance,
      defaultAlertElem,
      validationElems
    )(FormBuilderDocContext())

  //@XPathFunction
  def setControlLHHAMediatype(controlName: String, lhha: String, isHTML: Boolean): Unit =
    FormBuilder.setControlLhhatMediatype(controlName, lhha, isHTML)(FormBuilderDocContext())

  //@XPathFunction
  def setControlLabelHintHelpOrText(
    controlName : String,
    lhht        : String,
    value       : String,
    params      : Array[NodeInfo],
    isHTML      : Boolean
  ): Unit = {

    implicit val ctx = FormBuilderDocContext()

    val isOptionalLHHAT =
      lhht == LHHA.Help.entryName || lhht == fr.XMLNames.FRIterationLabelQName.localName

    // Make sure an optional element is present while we set content or attributes
    if (isOptionalLHHAT)
      FormBuilder.ensureCleanLHHAElements(controlName, lhht, count = 1, replace = true)

    FormBuilder.setControlLabelHintHelpOrText(controlName, lhht, value, Some(params), isHTML)

    // If an optional element turns out to be empty, then remove it
    if (isOptionalLHHAT) {

      val (doDelete, holders) =
        holdersToRemoveIfHasBlankOrMissingLHHAForAllLangs(controlName, FormBuilder.getControlLhhat(controlName, lhht).toList, lhht)

      if (doDelete) {
        delete(holders)
        delete(findControlByName(ctx.formDefinitionRootElem, controlName).toList child controlLHHATQName(lhht))
      }
    }
  }

  // Set the control's items for all languages
  //@XPathFunction
  def setControlItems(controlName: String, items: NodeInfo): Unit =
    FormBuilder.setControlItems(controlName, items)(FormBuilderDocContext())

  // For a given control, set the mediatype on the itemset labels to be HTML or plain text
  //@XPathFunction
  def setItemsetHTMLMediatype(controlName: String, isHTML: Boolean): Unit = {

    implicit val ctx = FormBuilderDocContext()

    if (isHTML != FormBuilder.isItemsetHTMLMediatype(controlName)) {
      val itemsetEl = FormRunner.findControlByName(ctx.formDefinitionRootElem, controlName).toList child "itemset"
      val labelHintEls = Seq("label", "hint") flatMap (itemsetEl.child(_))
      setHTMLMediatype(labelHintEls, isHTML)
    }
  }

  //@XPathFunction
  def isItemsetHTMLMediatype(controlName: String): Boolean =
    FormBuilder.isItemsetHTMLMediatype(controlName)(FormBuilderDocContext())

  //@XPathFunction
  def initializeGrids(doc: NodeInfo): Unit = {

    // Create context explicitly based on the document passed, as the node might not be
    // in the main Form Builder instance yet.
    implicit val ctx = FormBuilderDocContext(doc)

    // 1. Annotate all the grid and grid cells of the given document with unique ids,
    // if they don't have them already. We do this so that ids are stable as we move
    // things around, otherwise if the XForms document is recreated new automatic ids
    // are generated for objects without id.
    FormBuilder.annotateGridsAndCells(ctx.bodyElem)

    // 2. Select the first td if any
    ctx.bodyElem descendant GridTest descendant CellTest take 1 foreach selectCell
  }

  //@XPathFunction
  def findModelElem(doc: NodeInfo): NodeInfo =
    FormBuilderDocContext(doc).modelElem

  //@XPathFunction
  def findDataInstanceElem(doc: NodeInfo): NodeInfo =
    FormBuilderDocContext(doc).dataInstanceElem

  //@XPathFunction
  def findMetadataInstanceElem(doc: NodeInfo): NodeInfo =
    FormBuilderDocContext(doc).metadataInstanceElem

  // Create template content from a bind name
  // FIXME: Saxon can pass null as `bindings`.
  //@XPathFunction
  def createTemplateContentFromBindName(inDoc: NodeInfo, name: String, bindings: List[NodeInfo]): Option[NodeInfo] =
    FormBuilder.createTemplateContentFromBindName(name, Option(bindings) getOrElse Nil)(FormBuilderDocContext(inDoc))

  // See: https://github.com/orbeon/orbeon-forms/issues/633
  // See: https://github.com/orbeon/orbeon-forms/issues/3073
  //@XPathFunction
  def updateSectionTemplateContentHolders(inDoc: NodeInfo): Unit = {

    // Create context explicitly based on the document passed, as the node might not be
    // in the main Form Builder instance yet.
    implicit val ctx = FormBuilderDocContext(inDoc)

    // Find data holders for all section templates
    val holdersWithRoots =
      for {
        sectionNode   ← FormRunner.findSectionsWithTemplates(ctx.bodyElem)
        controlName   ← FormRunner.getControlNameOpt(sectionNode).toList
        holder        ← FormBuilder.findDataHolders(controlName) // TODO: What about within repeated sections? Templates ok?
        componentNode ← FormRunner.findComponentNodeForSection(sectionNode)
        xblNode       ← FormRunner.findXblXblForSectionTemplateNamespace(ctx.bodyElem, componentNode.namespaceURI)
        bindingNode   ← FormRunner.findXblBindingForLocalname(xblNode, componentNode.localname)
        instance      ← FormRunner.findXblInstance(bindingNode, fr.Names.FormTemplate)
        instanceRoot  ← instance / * headOption
      } yield
        holder → instanceRoot

    holdersWithRoots foreach { case (holder, instanceRoot) ⇒
      delete(holder / *)
      insert(into = holder, origin = instanceRoot / *)
    }
  }

  // Find all resource holders and elements which are unneeded because the resources are blank
  //@XPathFunction
  def findBlankHelpHoldersAndElements(inDoc: NodeInfo): Seq[NodeInfo] = {

    // Create context explicitly based on the document passed, as the node might not be
    // in the main Form Builder instance yet.
    implicit val ctx = FormBuilderDocContext(inDoc)

    val lhha = LHHA.Help

    val lhhaTest: Test = LHHA.QNameForValue(lhha)

    val allHelpElementsWithControlNames =
      ctx.bodyElem descendant lhhaTest map
      (lhhaElem ⇒ lhhaElem → lhhaElem.attValue("ref")) collect
      { case (lhhaElem, HelpRefMatcher(controlName)) ⇒ lhhaElem → controlName }

    allHelpElementsWithControlNames flatMap { case (lhhaElement, controlName) ⇒
      holdersToRemoveIfHasBlankOrMissingLHHAForAllLangs(controlName, List(lhhaElement), lhha.entryName)._2 :+ lhhaElement
    }
{'fraction_non_alphanumeric': 0.07922348243490081, 'fraction_numerical': 0.0020389957945711737, 'mean_word_length': 3.529049634474798, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 6, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '535692', 'n_tokens_mistral': 7053, 'n_tokens_neox': 6892, 'n_words': 1754}