<commit_before>// BSD License (http://lemurproject.org/galago-license)
package org.lemurproject.galago.core.retrieval.processing;

import gnu.trove.map.hash.TObjectIntHashMap;
import java.util.HashMap;
import org.lemurproject.galago.core.retrieval.iterator.CountIterator;
import org.lemurproject.galago.core.retrieval.iterator.StructuredIterator;
import org.lemurproject.galago.core.retrieval.query.Node;

/**
 *
 * @author irmarc
 */
public class FilteredStatisticsScoringContext extends ScoringContext
        implements ActiveContext {

  public TObjectIntHashMap<String> tfs;
  public TObjectIntHashMap<String> dfs;
  public long collectionLength = 0;
  public long documentCount = 0;
  HashMap<String, CountIterator> trackedIterators;

  public FilteredStatisticsScoringContext() {
    super();
    tfs = new TObjectIntHashMap<String>();
    dfs = new TObjectIntHashMap<String>();
    trackedIterators = new HashMap<String, CountIterator>();
  }

  @Override
  public void checkIterator(Node node, StructuredIterator iterator) {
    if (CountIterator.class.isAssignableFrom(iterator.getClass())) {
      trackedIterators.put(node.getDefaultParameter(),
              (CountIterator) iterator);
    }
  }
}<commit_msg>working on getting the last test to pass<commit_after>// BSD License (http://lemurproject.org/galago-license)
package org.lemurproject.galago.core.retrieval.processing;

import gnu.trove.map.hash.TObjectIntHashMap;
import java.util.HashMap;
import org.lemurproject.galago.core.retrieval.iterator.CountIterator;
import org.lemurproject.galago.core.retrieval.iterator.StructuredIterator;
import org.lemurproject.galago.core.retrieval.query.Node;

/**
 *
 * @author irmarc
 */
public class FilteredStatisticsScoringContext extends ScoringContext
        implements ActiveContext {

  public TObjectIntHashMap<String> tfs;
  public TObjectIntHashMap<String> dfs;
  public long collectionLength = 0;
  public long documentCount = 0;
  HashMap<String, CountIterator> trackedIterators;

  public FilteredStatisticsScoringContext() {
    super();
    tfs = new TObjectIntHashMap<String>();
    dfs = new TObjectIntHashMap<String>();
    trackedIterators = new HashMap<String, CountIterator>();
  }

  @Override
  public void checkIterator(Node node, StructuredIterator iterator) {
    String operator = node.getOperator();
    if (CountIterator.class.isAssignableFrom(iterator.getClass()) &&
            (operator.equals("counts") || operator.equals("extents"))) {
      trackedIterators.put(node.getDefaultParameter(),
              (CountIterator) iterator);
    }
  }
}
{'fraction_non_alphanumeric': 0.10479505027068832, 'fraction_numerical': 0.0015467904098994587, 'mean_word_length': 4.787472035794184, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3234405', 'n_tokens_mistral': 760, 'n_tokens_neox': 764, 'n_words': 146}