<commit_before>package is.hail.sparkextras

import is.hail.utils._
import org.apache.spark._
import org.apache.spark.rdd._
import org.apache.spark.ExposedUtils

import scala.reflect.ClassTag

class AssociativeCombiner[U](zero: U, combine: (U, U) => U) {
  case class TreeValue(var value: U, var end: Int)

  private val t = new java.util.TreeMap[Int, TreeValue]()

  def combine(i: Int, value0: U) {
    var value = value0
    var end = i

    val nexttv = t.get(i + 1)
    if (nexttv != null) {
      value = combine(value, nexttv.value)
      end = nexttv.end
      t.remove(i + 1)
    }

    val prevEntry = t.floorEntry(i - 1)
    if (prevEntry != null) {
      val prevtv = prevEntry.getValue
      if (prevtv.end == i - 1) {
        prevtv.value = combine(prevtv.value, value)
        prevtv.end = end<commit_msg>[hail] Log the size of the associative combiner queue (#6755)<commit_after>package is.hail.sparkextras

import is.hail.utils._
import org.apache.spark._
import org.apache.spark.rdd._
import org.apache.spark.ExposedUtils

import scala.reflect.ClassTag

class AssociativeCombiner[U](zero: U, combine: (U, U) => U) {
  case class TreeValue(var value: U, var end: Int)

  private val t = new java.util.TreeMap[Int, TreeValue]()

  def combine(i: Int, value0: U) {
    log.info(s"at result $i, AssociativeCombiner contains ${ t.size() } queued results")
    var value = value0
    var end = i

    val nexttv = t.get(i + 1)
    if (nexttv != null) {
      value = combine(value, nexttv.value)
      end = nexttv.end
      t.remove(i + 1)
    }

    val prevEntry = t.floorEntry(i - 1)
    if (prevEntry != null) {
      val prevtv = prevEntry.getValue
      if (prevtv.end == i - 1) {
        prevtv.value = combine(prevtv.value, value)
        prevtv.end = end
{'fraction_non_alphanumeric': 0.12619718309859154, 'fraction_numerical': 0.009014084507042254, 'mean_word_length': 2.844155844155844, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4040398', 'n_tokens_mistral': 651, 'n_tokens_neox': 625, 'n_words': 170}