<commit_before>package org.cdlib.was.ngIndexer;

import org.apache.solr.common._;
import org.apache.solr.client.solrj._;

class SolrDocumentCollection(val server : SolrServer,
                             val q : SolrQuery)
extends Iterable[SolrDocument] {

  /* you don't want to call this. */
  def apply (idx : Int) = this.iterator.toSeq(idx);

  lazy val length : Int =
    server.query(q.getCopy.setRows(0)).getResults.getNumFound.asInstanceOf[Int];

  override def toString = 
    iterator.peek.map(el=>"(%s, ...)".format(el)).getOrElse("(empty)");

  def iterator = new MyIterator[SolrDocument]() {
    var pos = 0;

    def fillCache {
      val results = server.query(q.getCopy.setStart(pos)).getResults;
      for (i <- new Range(0, results.size, 1)) {
        cache += results.get(i);
      }
      pos = (results.getStart.asInstanceOf[Int] + results.size.asInstanceOf[Int]);
    }
  }    
}<commit_msg>allow solrdocumentcollection to take any class that supports a query<commit_after>package org.cdlib.was.ngIndexer;

import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.params.SolrParams;

class SolrDocumentCollection(val server : { def query (q : SolrParams) : QueryResponse; },
                             val q : SolrQuery)
  extends Iterable[SolrDocument] {

  /* you don't want to call this. */
  def apply (idx : Int) = this.iterator.toSeq(idx);

  lazy val length : Int =
    server.query(q.getCopy.setRows(0)).getResults.getNumFound.asInstanceOf[Int];

  override def toString = 
    iterator.peek.map(el=>"(%s, ...)".format(el)).getOrElse("(empty)");

  def iterator = new MyIterator[SolrDocument]() {
    var pos = 0;

    def fillCache {
      val results = server.query(q.getCopy.setStart(pos)).getResults;
      for (i <- new Range(0, results.size, 1)) {
        cache += results.get(i);
      }
      pos = (results.getStart.asInstanceOf[Int] + results.size.asInstanceOf[Int]);
    }
  }    
}
{'fraction_non_alphanumeric': 0.14146341463414633, 'fraction_numerical': 0.003902439024390244, 'mean_word_length': 3.1771894093686353, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2015538', 'n_tokens_mistral': 706, 'n_tokens_neox': 680, 'n_words': 135}