<commit_before>package latis.util

/**
 * An Iterator that looks ahead and caches the next sample.
 * This makes it easier to know when we are done, especially 
 * when we are filtering. The original source may have another 
 * sample but this will keep looking for the next valid sample.
 * You can also "peek" at the next sample without advancing.
 */
abstract class PeekIterator[T >: Null] extends Iterator[T] {
  //Note, the bound on Null allows us to return null for generic type T.
  //TODO: implement something like takeWhile that leave the rest of the Iterator usable.
  
  /**
   * Cached next value. Will be null if there is no more elements.
   */
  private var _next: T = null

  /**
   * Take a look at the next sample without advancing to it.
   */
  final def peek: T = {
    _initialized //make sure we have cached the first value
    _next
  }
  
  /**
   * Use this lazy val so initialization will happen when this is first asked
   * if it is initialized.
   */
  private lazy val _initialized: Boolean = {
    _next = getNext
    true
  }
  
  /**
   * True if there is a cached next value.
   * The first call to this will cause the first value to be accessed and cached.
   */
  final def hasNext: Boolean = _initialized && _next != null

    
  /**
   * Return the 'next' value and cache the next 'next' value
   * to effectively advance to the next sample.
   */
  final def next: T = {
    _initialized //make sure we have cached the first value
    val current = _next
    _next = getNext //TODO: get next value to cache asynchronously?
    current
  }
  
  /**
   * Responsible for getting the next transformed item 
   * or null if there are no more valid items.
   * This should keep trying until a valid sample is found 
   * or it hits the end of the original iterator.
   */
  protected def getNext: T
}

object PeekIterator {
  
  /**
   * Wrap an Iterator so we can use it as a PeekIterator.
   */
  def apply[T >: Null](iterator: Iterator[T]) = new PeekIterator[T] {
    def getNext = {
      if (iterator.hasNext) iterator.next
      else null
    }
  }
}<commit_msg>throw error if constructed with a null iterator<commit_after>package latis.util

/**
 * An Iterator that looks ahead and caches the next sample.
 * This makes it easier to know when we are done, especially 
 * when we are filtering. The original source may have another 
 * sample but this will keep looking for the next valid sample.
 * You can also "peek" at the next sample without advancing.
 */
abstract class PeekIterator[T >: Null] extends Iterator[T] {
  //Note, the bound on Null allows us to return null for generic type T.
  //TODO: implement something like takeWhile that leave the rest of the Iterator usable.
  
  /**
   * Cached next value. Will be null if there is no more elements.
   */
  private var _next: T = null

  /**
   * Take a look at the next sample without advancing to it.
   */
  final def peek: T = {
    _initialized //make sure we have cached the first value
    _next
  }
  
  /**
   * Use this lazy val so initialization will happen when this is first asked
   * if it is initialized.
   */
  private lazy val _initialized: Boolean = {
    _next = getNext
    true
  }
  
  /**
   * True if there is a cached next value.
   * The first call to this will cause the first value to be accessed and cached.
   */
  final def hasNext: Boolean = _initialized && _next != null

    
  /**
   * Return the 'next' value and cache the next 'next' value
   * to effectively advance to the next sample.
   */
  final def next: T = {
    _initialized //make sure we have cached the first value
    val current = _next
    _next = getNext //TODO: get next value to cache asynchronously?
    current
  }
  
  /**
   * Responsible for getting the next transformed item 
   * or null if there are no more valid items.
   * This should keep trying until a valid sample is found 
   * or it hits the end of the original iterator.
   */
  protected def getNext: T
}

object PeekIterator {
  
  /**
   * Wrap an Iterator so we can use it as a PeekIterator.
   */
  def apply[T >: Null](iterator: Iterator[T]) = new PeekIterator[T] {
    if (iterator == null) throw new Error("PeekIterator can't wrap a null Iterator.")
    def getNext = {
      if (iterator.hasNext) iterator.next
      else null
    }
  }
}
{'fraction_non_alphanumeric': 0.07350901525658807, 'fraction_numerical': 0.0, 'mean_word_length': 2.6269907795473597, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3216152', 'n_tokens_mistral': 1248, 'n_tokens_neox': 1227, 'n_words': 602}