<commit_before>/* Copyright (c) 2011 The Regents of the University of California */

package org.cdlib.was.weari.solr;

import org.apache.solr.client.solrj.{SolrQuery,SolrServer};
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.params.SolrParams;

class SolrDocumentCollection(server : SolrServer, q : SolrQuery)
  extends Iterable[SolrDocument] {

  /* you don't want to call this. */
  def apply (idx : Int) = this.iterator.toSeq(idx);

  lazy val length : Int =
    server.query(q.getCopy.setRows(0)).getResults.getNumFound.asInstanceOf[Int];

  override def toString = 
    iterator.peek.map(el=>"(%s, ...)".format(el)).getOrElse("(empty)");

  def iterator = new CachingIterator[SolrDocument]() {
    var pos = 0;

    def fillCache {
      val results = server.query(q.getCopy.setStart(pos)).getResults;
      for (i <- new Range(0, results.size, 1)) {
        cache += results.get(i);
      }
      pos = (results.getStart.asInstanceOf[Int] + results.size.asInstanceOf[Int]);
    }
  }    
}<commit_msg>set length in solrdocumentcollection in fillcache<commit_after>/* Copyright (c) 2011 The Regents of the University of California */

package org.cdlib.was.weari.solr;

import org.apache.solr.client.solrj.{SolrQuery,SolrServer};
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.params.SolrParams;

class SolrDocumentCollection(server : SolrServer, q : SolrQuery)
  extends Iterable[SolrDocument] {

  /* you don't want to call this. */
  def apply (idx : Int) = this.iterator.toSeq(idx);

  override def toString = 
    iterator.peek.map(el=>"(%s, ...)".format(el)).getOrElse("(empty)");

  def iterator = new CachingIterator[SolrDocument]() {
    var pos = 0;

    var _length = -1;
    override def length = {
      /* if we haven't filled the cache yet, force it to be filled */
      if (_length == -1) this.peek;
      _length;
    }

    def fillCache {
      val results = server.query(q.getCopy.setStart(pos)).getResults;
      this._length = results.getNumFound.asInstanceOf[Int];
      for (i <- new Range(0, results.size, 1)) {
        cache += results.get(i);
      }
      pos = (results.getStart.asInstanceOf[Int] + results.size.asInstanceOf[Int]);
    }
  }    
}
{'fraction_non_alphanumeric': 0.13910093299406276, 'fraction_numerical': 0.00720949957591179, 'mean_word_length': 3.5019083969465647, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5937061', 'n_tokens_mistral': 826, 'n_tokens_neox': 786, 'n_words': 169}