<commit_before>/**
 * Copyright 2005 The Apache Software Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.nutch.searcher;

import org.apache.lucene.search.Searcher;
import org.apache.lucene.search.QueryFilter;
import org.apache.lucene.search.*;
import org.apache.lucene.index.Term;
import org.apache.lucene.misc.ChainedFilter;

import org.apache.hadoop.conf.Configuration;

import java.util.LinkedHashMap;
import java.util.Map;
import java.util.ArrayList;

import java.io.IOException;

/** Utility which converts certain query clauses into {@link QueryFilter}s and
 * caches these.  Only required clauses whose boost is zero are converted to
 * cached filters.  Range queries are converted to range filters.  This
 * accellerates query constraints like date, language, document format, etc.,
 * which do not affect ranking but might otherwise slow search considerably. */
class LuceneQueryOptimizer {

  private static class LimitExceeded extends RuntimeException {
    private int maxDoc;
    public LimitExceeded(int maxDoc) { this.maxDoc = maxDoc; }    
  }
  
  private static class LimitedCollector extends TopDocCollector {
    private int maxHits;
    
    public LimitedCollector(int numHits, int maxHits) {
      super(numHits);
      this.maxHits = maxHits;
    }

    public void collect(int doc, float score) {
      if (getTotalHits() >= maxHits)
        throw new LimitExceeded(doc);
      super.collect(doc, score);
    }
  }

  private LinkedHashMap cache;                   // an LRU cache of QueryFilter

  private float threshold;

  private int searcherMaxHits;

  /**
   * Construct an optimizer that caches and uses filters for required clauses
   * whose boost is zero.
   * 
   * @param cacheSize
   *          the number of QueryFilters to cache
   * @param threshold
   *          the fraction of documents which must contain a term
   */
  public LuceneQueryOptimizer(Configuration conf) {
    final int cacheSize = conf.getInt("searcher.filter.cache.size", 16);
    this.threshold = conf.getFloat("searcher.filter.cache.threshold",
        0.05f);
    this.searcherMaxHits = conf.getInt("searcher.max.hits", -1);
    this.searcherMaxHits = searcherMaxHits;
    this.cache = new LinkedHashMap(cacheSize, 0.75f, true) {
      protected boolean removeEldestEntry(Map.Entry eldest) {
        return size() > cacheSize; // limit size of cache
      }
    };
  }

  public TopDocs optimize(BooleanQuery original,
                          Searcher searcher, int numHits,
                          String sortField, boolean reverse)
    throws IOException {

    BooleanQuery query = new BooleanQuery();
    BooleanQuery cacheQuery = new BooleanQuery();
    BooleanQuery filterQuery = new BooleanQuery();
    ArrayList filters = new ArrayList();

    BooleanClause[] clauses = original.getClauses();
    for (int i = 0; i < clauses.length; i++) {
      BooleanClause c = clauses[i];
      if (c.isRequired()                          // required
          && c.getQuery().getBoost() == 0.0f) {   // boost is zero

        if (c.getQuery() instanceof TermQuery     // TermQuery
            && (searcher.docFreq(((TermQuery)c.getQuery()).getTerm())<commit_msg>rewmoved unneccessary assignment<commit_after>/**
 * Copyright 2005 The Apache Software Foundation
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package org.apache.nutch.searcher;

import org.apache.lucene.search.Searcher;
import org.apache.lucene.search.QueryFilter;
import org.apache.lucene.search.*;
import org.apache.lucene.index.Term;
import org.apache.lucene.misc.ChainedFilter;

import org.apache.hadoop.conf.Configuration;

import java.util.LinkedHashMap;
import java.util.Map;
import java.util.ArrayList;

import java.io.IOException;

/** Utility which converts certain query clauses into {@link QueryFilter}s and
 * caches these.  Only required clauses whose boost is zero are converted to
 * cached filters.  Range queries are converted to range filters.  This
 * accellerates query constraints like date, language, document format, etc.,
 * which do not affect ranking but might otherwise slow search considerably. */
class LuceneQueryOptimizer {

  private static class LimitExceeded extends RuntimeException {
    private int maxDoc;
    public LimitExceeded(int maxDoc) { this.maxDoc = maxDoc; }    
  }
  
  private static class LimitedCollector extends TopDocCollector {
    private int maxHits;
    
    public LimitedCollector(int numHits, int maxHits) {
      super(numHits);
      this.maxHits = maxHits;
    }

    public void collect(int doc, float score) {
      if (getTotalHits() >= maxHits)
        throw new LimitExceeded(doc);
      super.collect(doc, score);
    }
  }

  private LinkedHashMap cache;                   // an LRU cache of QueryFilter

  private float threshold;

  private int searcherMaxHits;

  /**
   * Construct an optimizer that caches and uses filters for required clauses
   * whose boost is zero.
   * 
   * @param cacheSize
   *          the number of QueryFilters to cache
   * @param threshold
   *          the fraction of documents which must contain a term
   */
  public LuceneQueryOptimizer(Configuration conf) {
    final int cacheSize = conf.getInt("searcher.filter.cache.size", 16);
    this.threshold = conf.getFloat("searcher.filter.cache.threshold",
        0.05f);
    this.searcherMaxHits = conf.getInt("searcher.max.hits", -1);
    this.cache = new LinkedHashMap(cacheSize, 0.75f, true) {
      protected boolean removeEldestEntry(Map.Entry eldest) {
        return size() > cacheSize; // limit size of cache
      }
    };
  }

  public TopDocs optimize(BooleanQuery original,
                          Searcher searcher, int numHits,
                          String sortField, boolean reverse)
    throws IOException {

    BooleanQuery query = new BooleanQuery();
    BooleanQuery cacheQuery = new BooleanQuery();
    BooleanQuery filterQuery = new BooleanQuery();
    ArrayList filters = new ArrayList();

    BooleanClause[] clauses = original.getClauses();
    for (int i = 0; i < clauses.length; i++) {
      BooleanClause c = clauses[i];
      if (c.isRequired()                          // required
          && c.getQuery().getBoost() == 0.0f) {   // boost is zero

        if (c.getQuery() instanceof TermQuery     // TermQuery
            && (searcher.docFreq(((TermQuery)c.getQuery()).getTerm())
{'fraction_non_alphanumeric': 0.08801725299905648, 'fraction_numerical': 0.005391562205148942, 'mean_word_length': 3.0524303659202623, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6718723', 'n_tokens_mistral': 2078, 'n_tokens_neox': 2021, 'n_words': 685}