<commit_before>package org.scalaide.ui.internal.editor

import org.eclipse.core.runtime.NullProgressMonitor
import org.eclipse.jdt.core.IJavaElement
import org.eclipse.jdt.core.IJavaProject
import org.eclipse.jdt.core.ITypeRoot
import org.eclipse.jdt.internal.ui.JavaPlugin
import org.eclipse.jdt.internal.ui.javaeditor.IClassFileEditorInput
import org.eclipse.jdt.internal.ui.javaeditor.ICompilationUnitDocumentProvider
import org.eclipse.jdt.internal.ui.text.java.SmartSemicolonAutoEditStrategy
import org.eclipse.jdt.ui.text.IJavaPartitions
import org.eclipse.jdt.ui.text.JavaSourceViewerConfiguration
import org.eclipse.jface.internal.text.html.BrowserInformationControl
import org.eclipse.jface.internal.text.html.HTMLPrinter
import org.eclipse.jface.preference.IPreferenceStore
import org.eclipse.jface.text.AbstractReusableInformationControlCreator
import org.eclipse.jface.text.DefaultInformationControl
import org.eclipse.jface.text.DefaultTextHover
import org.eclipse.jface.text.IAutoEditStrategy
import org.eclipse.jface.text.IDocument
import org.eclipse.jface.text.IInformationControl
import org.eclipse.jface.text.ITextHover
import org.eclipse.jface.text.formatter.IContentFormatter
import org.eclipse.jface.text.formatter.MultiPassContentFormatter
import org.eclipse.jface.text.hyperlink.IHyperlinkDetector
import org.eclipse.jface.text.hyperlink.URLHyperlinkDetector
import org.eclipse.jface.text.information.InformationPresenter
import org.eclipse.jface.text.reconciler.IReconciler
import org.eclipse.jface.text.reconciler.MonoReconciler
import org.eclipse.jface.text.rules.DefaultDamagerRepairer
import org.eclipse.jface.text.source.Annotation
import org.eclipse.jface.text.source.DefaultAnnotationHover
import org.eclipse.jface.text.source.ISourceViewer
import org.eclipse.jface.util.PropertyChangeEvent
import org.eclipse.swt.widgets.Shell
import org.eclipse.ui.texteditor.ChainedPreferenceStore
import org.scalaide.core.ScalaPlugin
import org.scalaide.core.hyperlink.detector.CompositeHyperlinkDetector
import org.scalaide.core.hyperlink.detector.DeclarationHyperlinkDetector
import org.scalaide.core.hyperlink.detector.ImplicitHyperlinkDetector
import org.scalaide.core.internal.formatter.FormatterPreferences._
import org.scalaide.core.internal.formatter.ScalaFormattingStrategy
import org.scalaide.core.internal.jdt.model.ScalaCompilationUnit
import org.scalaide.core.internal.lexical._
import org.scalaide.ui.editor.extensionpoints.ScalaHoverDebugOverrideExtensionPoint
import org.scalaide.ui.internal.editor.autoedits._
import org.scalaide.ui.internal.editor.hover.BrowserControlAdditions
import org.scalaide.ui.internal.editor.hover.HoverInformationProvider
import org.scalaide.ui.internal.editor.hover.HtmlHover
import org.scalaide.ui.internal.editor.hover.ScalaHover
import org.scalaide.ui.internal.reconciliation.ScalaReconcilingStrategy
import org.scalaide.ui.syntax.{ScalaSyntaxClasses => SSC}

import scalariform.ScalaVersions
import scalariform.formatter.preferences._

class ScalaSourceViewerConfiguration(
  javaPreferenceStore: IPreferenceStore,
  scalaPreferenceStore: IPreferenceStore,
  editor: ScalaEditor)
    extends JavaSourceViewerConfiguration(
      JavaPlugin.getDefault.getJavaTextTools.getColorManager,
      javaPreferenceStore,
      editor,
      IJavaPartitions.JAVA_PARTITIONING) {

  private val combinedPrefStore = new ChainedPreferenceStore(
      Array(scalaPreferenceStore, javaPreferenceStore))

  private val codeHighlightingScanners = {
    val scalaCodeScanner = new ScalaCodeScanner(scalaPreferenceStore, ScalaVersions.DEFAULT)
    val singleLineCommentScanner = new ScalaCommentScanner(scalaPreferenceStore, javaPreferenceStore, SSC.SINGLE_LINE_COMMENT, SSC.TASK_TAG)
    val multiLineCommentScanner = new ScalaCommentScanner(scalaPreferenceStore, javaPreferenceStore, SSC.MULTI_LINE_COMMENT, SSC.TASK_TAG)
    val scaladocScanner = new ScaladocTokenScanner(scalaPreferenceStore, javaPreferenceStore, SSC.SCALADOC, SSC.SCALADOC_ANNOTATION, SSC.SCALADOC_MACRO, SSC.TASK_TAG)
    val scaladocCodeBlockScanner = new SingleTokenScanner(scalaPreferenceStore, SSC.SCALADOC_CODE_BLOCK)
    val stringScanner = new StringTokenScanner(scalaPreferenceStore, SSC.ESCAPE_SEQUENCE, SSC.STRING)
    val characterScanner = new StringTokenScanner(scalaPreferenceStore, SSC.ESCAPE_SEQUENCE, SSC.CHARACTER)
    val multiLineStringScanner = new SingleTokenScanner(scalaPreferenceStore, SSC.MULTI_LINE_STRING)
    val xmlTagScanner = new XmlTagScanner(scalaPreferenceStore)
    val xmlCommentScanner = new XmlCommentScanner(scalaPreferenceStore)
    val xmlCDATAScanner = new XmlCDATAScanner(scalaPreferenceStore)
    val xmlPCDATAScanner = new SingleTokenScanner(scalaPreferenceStore, SSC.DEFAULT)
    val xmlPIScanner = new XmlPIScanner(scalaPreferenceStore)

    Map(
      IDocument.DEFAULT_CONTENT_TYPE -> scalaCodeScanner,
      IJavaPartitions.JAVA_DOC -> scaladocScanner,
      ScalaPartitions.SCALADOC_CODE_BLOCK -> scaladocCodeBlockScanner,
      IJavaPartitions.JAVA_SINGLE_LINE_COMMENT -> singleLineCommentScanner,
      IJavaPartitions.JAVA_MULTI_LINE_COMMENT -> multiLineCommentScanner,
      IJavaPartitions.JAVA_STRING -> stringScanner,
      IJavaPartitions.JAVA_CHARACTER -> characterScanner,
      ScalaPartitions.SCALA_MULTI_LINE_STRING -> multiLineStringScanner,
      ScalaPartitions.XML_TAG -> xmlTagScanner,
      ScalaPartitions.XML_COMMENT -> xmlCommentScanner,
      ScalaPartitions.XML_CDATA -> xmlCDATAScanner,
      ScalaPartitions.XML_PCDATA -> xmlPCDATAScanner,
      ScalaPartitions.XML_PI -> xmlPIScanner
    )
  }

  override def getTabWidth(sourceViewer: ISourceViewer): Int =
    scalaPreferenceStore.getInt(IndentSpaces.eclipseKey)

  /**
   * Indent prefixes are all possible variations of strings of a given
   * 'indent' length that can be inserted as indent.
   *
   * As an example, when the indent depth is 4, these are the prefixes:
   *
   * '\t', ' \t', '  \t', '   \t', '    ', '' (when only tabs should be inserted)
   * '    ', '\t', ' \t', '  \t', '   \t', '' (when only spaces should be inserted)
   *
   * The array always contains 2 + indent depth elements, where the last element
   * is always the empty string. The first element describes a full indent depth,
   * whereas the remaining elements describe a combination of spaces + a tab to
   * fill a full indent depth.
   */
  override def getIndentPrefixes(sourceViewer: ISourceViewer, contentType: String): Array[String] = {
    val spaceWidth = scalaPreferenceStore.getInt(IndentSpaces.eclipseKey)
    val useTabs = scalaPreferenceStore.getBoolean(IndentWithTabs.eclipseKey)

    val spacePrefix = " " * spaceWidth
    val prefixes = 0 until spaceWidth map (i => " " * i + "\t")

    if (useTabs)
      (prefixes :+ spacePrefix :+ "").toArray
    else
      (spacePrefix +: prefixes :+ "").toArray
  }

  /**
   * This annotation hover needs to trim error messages to a single line due to
   * some stylistic limitations in the logic that computes the size of the
   * hover. The method that computes the size is:
   *
   * [[org.eclipse.jface.internal.text.html.BrowserInformationControl.computeSizeHint()]]
   *
   * It basically converts HTML to text whose size is then considered. This
   * works only for a very basic form of HTML - CSS specifications like margins
   * can not be considered. Therefore, we need to remove as much text as
   * possible and show only single lines to keep the error rate of line
   * measurements low.
   *
   * This is also what the JDT is doing - but they don't even use HTML but only
   * the converted text form. This way they don't have any stylistic problems
   * but they also can't apply there Java hover specific configurations
   * anymore.
   */
  private val annotationHover = new DefaultAnnotationHover(/* showLineNumber */ false) with HtmlHover {
    import HTMLPrinter._

    override def isIncluded(a: Annotation) =
      isShowInVerticalRuler(a)

    val UnimplementedMembers = """(class .* needs to be abstract, since:\W*it has \d+ unimplemented members\.)[\S\s]*""".r

    val msgFormatter: String => String = {
      case UnimplementedMembers(errorMsg) =>
        convertContentToHtml(errorMsg)
      case str =>
        convertContentToHtml(str)
    }

    override def formatSingleMessage(msg: String) = {
      createHtmlOutput { sb =>
        sb.append(msgFormatter(msg))
      }
    }

    override def formatMultipleMessages(msgs: java.util.List[_]) = {
      createHtmlOutput { sb =>
        import HTMLPrinter._
        import collection.JavaConverters._
        addParagraph(sb, "Multiple markers at this line:")
        startBulletList(sb)
        msgs.asScala foreach (msg => addBullet(sb, msgFormatter(msg.asInstanceOf[String])))
        endBulletList(sb)
      }
    }
  }

  override def getInformationControlCreator(sourceViewer: ISourceViewer) = new AbstractReusableInformationControlCreator {
    override def doCreateInformationControl(parent: Shell): IInformationControl = {
      if (BrowserInformationControl.isAvailable(parent))
        new BrowserInformationControl(parent, ScalaHover.HoverFontId, /* resizable */ false) with BrowserControlAdditions
      else
        new DefaultInformationControl(parent, /* resizable */ false)
    }
  }

  override def getInformationPresenter(sourceViewer: ISourceViewer) = {
    val ip = new HoverInformationProvider(compilationUnit map (new ScalaHover(_)))
    val p = new InformationPresenter(ip.getInformationPresenterControlCreator())

    p.setDocumentPartitioning(getConfiguredDocumentPartitioning(sourceViewer))
    getConfiguredContentTypes(sourceViewer) foreach (p.setInformationProvider(ip, _))
    p
  }

  override def getOverviewRulerAnnotationHover(sourceViewer: ISourceViewer) = annotationHover

  override def getAnnotationHover(sourceViewer: ISourceViewer) = annotationHover

  override def getReconciler(sourceViewer: ISourceViewer): IReconciler =
    if (editor ne null) {
      val reconciler = new MonoReconciler(new ScalaReconcilingStrategy(editor), /*isIncremental = */ false)
      // FG: I don't know any better that to defer this to the MonoReconciler constructor's default value
      // reconciler.setDelay(500)
      reconciler.install(sourceViewer)
      reconciler.setProgressMonitor(new NullProgressMonitor())
      reconciler
    } else
      null // the editor is null for the Syntax coloring previewer pane, (so no reconciliation)

  override def getPresentationReconciler(sourceViewer: ISourceViewer): ScalaPresentationReconciler = {
    val reconciler = new ScalaPresentationReconciler()
    reconciler.setDocumentPartitioning(getConfiguredDocumentPartitioning(sourceViewer))

    for ((partitionType, tokenScanner) <- codeHighlightingScanners) {
      val dr = new DefaultDamagerRepairer(tokenScanner)<commit_msg>Show HTML hover when an annotation hover is focused<commit_after>package org.scalaide.ui.internal.editor

import org.eclipse.core.runtime.NullProgressMonitor
import org.eclipse.jdt.core.IJavaElement
import org.eclipse.jdt.core.IJavaProject
import org.eclipse.jdt.core.ITypeRoot
import org.eclipse.jdt.internal.ui.JavaPlugin
import org.eclipse.jdt.internal.ui.javaeditor.IClassFileEditorInput
import org.eclipse.jdt.internal.ui.javaeditor.ICompilationUnitDocumentProvider
import org.eclipse.jdt.internal.ui.text.java.SmartSemicolonAutoEditStrategy
import org.eclipse.jdt.ui.text.IJavaPartitions
import org.eclipse.jdt.ui.text.JavaSourceViewerConfiguration
import org.eclipse.jface.internal.text.html.BrowserInformationControl
import org.eclipse.jface.internal.text.html.HTMLPrinter
import org.eclipse.jface.preference.IPreferenceStore
import org.eclipse.jface.text.AbstractReusableInformationControlCreator
import org.eclipse.jface.text.DefaultInformationControl
import org.eclipse.jface.text.DefaultTextHover
import org.eclipse.jface.text.IAutoEditStrategy
import org.eclipse.jface.text.IDocument
import org.eclipse.jface.text.IInformationControl
import org.eclipse.jface.text.IInformationControlCreator
import org.eclipse.jface.text.ITextHover
import org.eclipse.jface.text.formatter.IContentFormatter
import org.eclipse.jface.text.formatter.MultiPassContentFormatter
import org.eclipse.jface.text.hyperlink.IHyperlinkDetector
import org.eclipse.jface.text.hyperlink.URLHyperlinkDetector
import org.eclipse.jface.text.information.InformationPresenter
import org.eclipse.jface.text.reconciler.IReconciler
import org.eclipse.jface.text.reconciler.MonoReconciler
import org.eclipse.jface.text.rules.DefaultDamagerRepairer
import org.eclipse.jface.text.source.Annotation
import org.eclipse.jface.text.source.DefaultAnnotationHover
import org.eclipse.jface.text.source.IAnnotationHoverExtension
import org.eclipse.jface.text.source.ILineRange
import org.eclipse.jface.text.source.ISourceViewer
import org.eclipse.jface.text.source.LineRange
import org.eclipse.jface.util.PropertyChangeEvent
import org.eclipse.swt.widgets.Shell
import org.eclipse.ui.texteditor.ChainedPreferenceStore
import org.scalaide.core.ScalaPlugin
import org.scalaide.core.hyperlink.detector.CompositeHyperlinkDetector
import org.scalaide.core.hyperlink.detector.DeclarationHyperlinkDetector
import org.scalaide.core.hyperlink.detector.ImplicitHyperlinkDetector
import org.scalaide.core.internal.formatter.FormatterPreferences._
import org.scalaide.core.internal.formatter.ScalaFormattingStrategy
import org.scalaide.core.internal.jdt.model.ScalaCompilationUnit
import org.scalaide.core.internal.lexical._
import org.scalaide.ui.editor.extensionpoints.ScalaHoverDebugOverrideExtensionPoint
import org.scalaide.ui.internal.editor.autoedits._
import org.scalaide.ui.internal.editor.hover.BrowserControlAdditions
import org.scalaide.ui.internal.editor.hover.HoverInformationProvider
import org.scalaide.ui.internal.editor.hover.HtmlHover
import org.scalaide.ui.internal.editor.hover.ScalaHover
import org.scalaide.ui.internal.reconciliation.ScalaReconcilingStrategy
import org.scalaide.ui.syntax.{ScalaSyntaxClasses => SSC}

import scalariform.ScalaVersions
import scalariform.formatter.preferences._

class ScalaSourceViewerConfiguration(
  javaPreferenceStore: IPreferenceStore,
  scalaPreferenceStore: IPreferenceStore,
  editor: ScalaEditor)
    extends JavaSourceViewerConfiguration(
      JavaPlugin.getDefault.getJavaTextTools.getColorManager,
      javaPreferenceStore,
      editor,
      IJavaPartitions.JAVA_PARTITIONING) {

  private val combinedPrefStore = new ChainedPreferenceStore(
      Array(scalaPreferenceStore, javaPreferenceStore))

  private val codeHighlightingScanners = {
    val scalaCodeScanner = new ScalaCodeScanner(scalaPreferenceStore, ScalaVersions.DEFAULT)
    val singleLineCommentScanner = new ScalaCommentScanner(scalaPreferenceStore, javaPreferenceStore, SSC.SINGLE_LINE_COMMENT, SSC.TASK_TAG)
    val multiLineCommentScanner = new ScalaCommentScanner(scalaPreferenceStore, javaPreferenceStore, SSC.MULTI_LINE_COMMENT, SSC.TASK_TAG)
    val scaladocScanner = new ScaladocTokenScanner(scalaPreferenceStore, javaPreferenceStore, SSC.SCALADOC, SSC.SCALADOC_ANNOTATION, SSC.SCALADOC_MACRO, SSC.TASK_TAG)
    val scaladocCodeBlockScanner = new SingleTokenScanner(scalaPreferenceStore, SSC.SCALADOC_CODE_BLOCK)
    val stringScanner = new StringTokenScanner(scalaPreferenceStore, SSC.ESCAPE_SEQUENCE, SSC.STRING)
    val characterScanner = new StringTokenScanner(scalaPreferenceStore, SSC.ESCAPE_SEQUENCE, SSC.CHARACTER)
    val multiLineStringScanner = new SingleTokenScanner(scalaPreferenceStore, SSC.MULTI_LINE_STRING)
    val xmlTagScanner = new XmlTagScanner(scalaPreferenceStore)
    val xmlCommentScanner = new XmlCommentScanner(scalaPreferenceStore)
    val xmlCDATAScanner = new XmlCDATAScanner(scalaPreferenceStore)
    val xmlPCDATAScanner = new SingleTokenScanner(scalaPreferenceStore, SSC.DEFAULT)
    val xmlPIScanner = new XmlPIScanner(scalaPreferenceStore)

    Map(
      IDocument.DEFAULT_CONTENT_TYPE -> scalaCodeScanner,
      IJavaPartitions.JAVA_DOC -> scaladocScanner,
      ScalaPartitions.SCALADOC_CODE_BLOCK -> scaladocCodeBlockScanner,
      IJavaPartitions.JAVA_SINGLE_LINE_COMMENT -> singleLineCommentScanner,
      IJavaPartitions.JAVA_MULTI_LINE_COMMENT -> multiLineCommentScanner,
      IJavaPartitions.JAVA_STRING -> stringScanner,
      IJavaPartitions.JAVA_CHARACTER -> characterScanner,
      ScalaPartitions.SCALA_MULTI_LINE_STRING -> multiLineStringScanner,
      ScalaPartitions.XML_TAG -> xmlTagScanner,
      ScalaPartitions.XML_COMMENT -> xmlCommentScanner,
      ScalaPartitions.XML_CDATA -> xmlCDATAScanner,
      ScalaPartitions.XML_PCDATA -> xmlPCDATAScanner,
      ScalaPartitions.XML_PI -> xmlPIScanner
    )
  }

  override def getTabWidth(sourceViewer: ISourceViewer): Int =
    scalaPreferenceStore.getInt(IndentSpaces.eclipseKey)

  /**
   * Indent prefixes are all possible variations of strings of a given
   * 'indent' length that can be inserted as indent.
   *
   * As an example, when the indent depth is 4, these are the prefixes:
   *
   * '\t', ' \t', '  \t', '   \t', '    ', '' (when only tabs should be inserted)
   * '    ', '\t', ' \t', '  \t', '   \t', '' (when only spaces should be inserted)
   *
   * The array always contains 2 + indent depth elements, where the last element
   * is always the empty string. The first element describes a full indent depth,
   * whereas the remaining elements describe a combination of spaces + a tab to
   * fill a full indent depth.
   */
  override def getIndentPrefixes(sourceViewer: ISourceViewer, contentType: String): Array[String] = {
    val spaceWidth = scalaPreferenceStore.getInt(IndentSpaces.eclipseKey)
    val useTabs = scalaPreferenceStore.getBoolean(IndentWithTabs.eclipseKey)

    val spacePrefix = " " * spaceWidth
    val prefixes = 0 until spaceWidth map (i => " " * i + "\t")

    if (useTabs)
      (prefixes :+ spacePrefix :+ "").toArray
    else
      (spacePrefix +: prefixes :+ "").toArray
  }

  /**
   * This annotation hover needs to trim error messages to a single line due to
   * some stylistic limitations in the logic that computes the size of the
   * hover. The method that computes the size is:
   *
   * [[org.eclipse.jface.internal.text.html.BrowserInformationControl.computeSizeHint()]]
   *
   * It basically converts HTML to text whose size is then considered. This
   * works only for a very basic form of HTML - CSS specifications like margins
   * can not be considered. Therefore, we need to remove as much text as
   * possible and show only single lines to keep the error rate of line
   * measurements low.
   *
   * This is also what the JDT is doing - but they don't even use HTML but only
   * the converted text form. This way they don't have any stylistic problems
   * but they also can't apply there Java hover specific configurations
   * anymore.
   */
  private val annotationHover = new DefaultAnnotationHover(/* showLineNumber */ false)
      with IAnnotationHoverExtension
      with HtmlHover {
    import HTMLPrinter._

    override def isIncluded(a: Annotation) =
      isShowInVerticalRuler(a)

    val UnimplementedMembers = """(class .* needs to be abstract, since:\W*it has \d+ unimplemented members\.)[\S\s]*""".r

    val msgFormatter: String => String = {
      case UnimplementedMembers(errorMsg) =>
        convertContentToHtml(errorMsg)
      case str =>
        convertContentToHtml(str)
    }

    override def formatSingleMessage(msg: String) = {
      createHtmlOutput { sb =>
        sb.append(msgFormatter(msg))
      }
    }

    override def formatMultipleMessages(msgs: java.util.List[_]) = {
      createHtmlOutput { sb =>
        import HTMLPrinter._
        import collection.JavaConverters._
        addParagraph(sb, "Multiple markers at this line:")
        startBulletList(sb)
        msgs.asScala foreach (msg => addBullet(sb, msgFormatter(msg.asInstanceOf[String])))
        endBulletList(sb)
      }
    }

    override def canHandleMouseCursor(): Boolean =
      false

    override def getHoverControlCreator(): IInformationControlCreator = new AbstractReusableInformationControlCreator {
      override def doCreateInformationControl(parent: Shell): IInformationControl = {
        if (BrowserInformationControl.isAvailable(parent))
          new BrowserInformationControl(parent, ScalaHover.HoverFontId, /* resizable */ false) with BrowserControlAdditions
        else
          new DefaultInformationControl(parent, /* resizable */ false)
      }
    }

    override def getHoverInfo(sourceViewer: ISourceViewer, lineRange: ILineRange, visibleNumberOfLines: Int): Object =
      getHoverInfo(sourceViewer, lineRange.getStartLine())

    override def getHoverLineRange(sourceViewer: ISourceViewer, lineNumber: Int): ILineRange =
      new LineRange(lineNumber, /* numberOfLines */ 1)

  }

  override def getInformationControlCreator(sourceViewer: ISourceViewer) =
    annotationHover.getHoverControlCreator()

  override def getInformationPresenter(sourceViewer: ISourceViewer) = {
    val ip = new HoverInformationProvider(compilationUnit map (new ScalaHover(_)))
    val p = new InformationPresenter(ip.getInformationPresenterControlCreator())

    p.setDocumentPartitioning(getConfiguredDocumentPartitioning(sourceViewer))
    getConfiguredContentTypes(sourceViewer) foreach (p.setInformationProvider(ip, _))
    p
  }

  override def getOverviewRulerAnnotationHover(sourceViewer: ISourceViewer) = annotationHover

  override def getAnnotationHover(sourceViewer: ISourceViewer) = annotationHover

  override def getReconciler(sourceViewer: ISourceViewer): IReconciler =
    if (editor ne null) {
      val reconciler = new MonoReconciler(new ScalaReconcilingStrategy(editor), /*isIncremental = */ false)
      // FG: I don't know any better that to defer this to the MonoReconciler constructor's default value
      // reconciler.setDelay(500)
      reconciler.install(sourceViewer)
      reconciler.setProgressMonitor(new NullProgressMonitor())
      reconciler
    } else
      null // the editor is null for the Syntax coloring previewer pane, (so no reconciliation)

  override def getPresentationReconciler(sourceViewer: ISourceViewer): ScalaPresentationReconciler = {
    val reconciler = new ScalaPresentationReconciler()
    reconciler.setDocumentPartitioning(getConfiguredDocumentPartitioning(sourceViewer))

    for ((partitionType, tokenScanner) <- codeHighlightingScanners) {
      val dr = new DefaultDamagerRepairer(tokenScanner)
{'fraction_non_alphanumeric': 0.07871668224091131, 'fraction_numerical': 0.0005784719441107106, 'mean_word_length': 4.764042062067197, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 20}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6652690', 'n_tokens_mistral': 6804, 'n_tokens_neox': 6578, 'n_words': 1415}