<commit_before>package org.globalnames.parser

import java.io.Writer

import org.apache.commons.lang3.text.translate._

import scala.collection.mutable.ArrayBuffer

class TrackingPositionsUnescapeHtml4Translator extends AggregateTranslator {
  private final val translators =
    List(EntityArrays.BASIC_UNESCAPE,
         EntityArrays.ISO8859_1_UNESCAPE,
         EntityArrays.HTML40_EXTENDED_UNESCAPE()).map { strss =>
    new LookupTranslator(strss.map { _.map { _.asInstanceOf[CharSequence] }}: _*)
  } :+ new NumericEntityUnescaper

  private final val positions = ArrayBuffer.fill(1)(0)
  private final var identity = true

  override def translate(input: CharSequence, index: Int, out: Writer): Int = {
    val consumed = translators.foldLeft(0) { (cnsmed, cst) =>
      if (cnsmed == 0) cst.translate(input, index, out)
      else cnsmed
    }
    positions += positions.last + (if (consumed == 0) 1<commit_msg>= Fix cast to Array[Array[CharSequence]]<commit_after>package org.globalnames.parser

import java.io.Writer

import org.apache.commons.lang3.text.translate._

import scala.collection.mutable.ArrayBuffer

class TrackingPositionsUnescapeHtml4Translator extends AggregateTranslator {
  private final val translators =
    List(EntityArrays.BASIC_UNESCAPE,
         EntityArrays.ISO8859_1_UNESCAPE,
         EntityArrays.HTML40_EXTENDED_UNESCAPE()).map { strss =>
    new LookupTranslator(strss.asInstanceOf[Array[Array[CharSequence]]]: _*)
  } :+ new NumericEntityUnescaper

  private final val positions = ArrayBuffer.fill(1)(0)
  private final var identity = true

  override def translate(input: CharSequence, index: Int, out: Writer): Int = {
    val consumed = translators.foldLeft(0) { (cnsmed, cst) =>
      if (cnsmed == 0) cst.translate(input, index, out)
      else cnsmed
    }
    positions += positions.last + (if (consumed == 0) 1
{'fraction_non_alphanumeric': 0.10161290322580645, 'fraction_numerical': 0.016129032258064516, 'mean_word_length': 4.347701149425287, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4336613', 'n_tokens_mistral': 607, 'n_tokens_neox': 596, 'n_words': 129}