<commit_before>package org.apache.lucene.search;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


import java.util.Random;

import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;

import static org.junit.Assert.*;

/** Test BooleanQuery2 against BooleanQuery by overriding the standard query parser.
 * This also tests the scoring order of BooleanQuery.
 */
public class TestBoolean2 extends LuceneTestCase {
  private static IndexSearcher searcher;
  private static IndexSearcher bigSearcher;
  private static IndexReader reader;
  private static int NUM_EXTRA_DOCS = 6000;

  public static final String field = "field";
  private static Directory directory;
  private static Directory dir2;
  private static int mulFactor;

  @BeforeClass
  public static void beforeClass() throws Exception {
    directory = newDirectory();
    RandomIndexWriter writer= new RandomIndexWriter(random, directory);
    for (int i = 0; i < docFields.length; i++) {
      Document doc = new Document();
      doc.add(newField(field, docFields[i], Field.Store.NO, Field.Index.ANALYZED));
      writer.addDocument(doc);
    }
    writer.close();
    searcher = new IndexSearcher(directory, true);

    // Make big index
    dir2 = new MockDirectoryWrapper(new RAMDirectory(directory));

    // First multiply small test index:
    mulFactor = 1;
    int docCount = 0;
    do {
      final Directory copy = new MockDirectoryWrapper(new RAMDirectory(dir2));
      RandomIndexWriter w = new RandomIndexWriter(random, dir2);
      w.addIndexes(new Directory[] {copy});
      docCount = w.maxDoc();
      w.close();
      mulFactor *= 2;
    } while(docCount < 3000);

    RandomIndexWriter w = new RandomIndexWriter(random, dir2, 
        newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))
        .setMaxBufferedDocs(_TestUtil.nextInt(random, 50, 1000)));
    Document doc = new Document();
    doc.add(newField("field2", "xxx", Field.Store.NO, Field.Index.ANALYZED));
    for(int i=0;i<NUM_EXTRA_DOCS/2;i++) {
      w.addDocument(doc);
    }
    doc = new Document();
    doc.add(newField("field2", "big bad bug", Field.Store.NO, Field.Index.ANALYZED));
    for(int i=0;i<NUM_EXTRA_DOCS/2;i++) {
      w.addDocument(doc);
    }
    reader = w.getReader();
    bigSearcher = new IndexSearcher(reader);
    w.close();
  }

  @AfterClass
  public static void afterClass() throws Exception {
    searcher.close();
    reader.close();
    dir2.close();
    directory.close();
    bigSearcher.close();
    searcher = null;
    reader = null;
    dir2 = null;
    directory = null;
    bigSearcher = null;
  }

  private static String[] docFields = {
    "w1 w2 w3 w4 w5",
    "w1 w3 w2 w3",
    "w1 xx w2 yy w3",
    "w1 w3 xx w2 yy w3"
  };

  public Query makeQuery(String queryText) throws ParseException {
    Query q = (new QueryParser(TEST_VERSION_CURRENT, field, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))).parse(queryText);
    return q;
  }

  public void queriesTest(String queryText, int[] expDocNrs) throws Exception {
//System.out.println();
//System.out.println("Query: " + queryText);

    Query query1 = makeQuery(queryText);
    TopScoreDocCollector collector = TopScoreDocCollector.create(1000, false);
    searcher.search(query1, null, collector);
    ScoreDoc[] hits1 = collector.topDocs().scoreDocs;
    
    Query query2 = makeQuery(queryText); // there should be no need to parse again...
    collector = TopScoreDocCollector.create(1000, true);
    searcher.search(query2, null, collector);
    ScoreDoc[] hits2 = collector.topDocs().scoreDocs; 

    assertEquals(mulFactor * collector.totalHits,
                 bigSearcher.search(query1, 1).totalHits);
      
    CheckHits.checkHitsQuery(query2, hits1, hits2, expDocNrs);
  }

  @Test
  public void testQueries01() throws Exception {
    String queryText = "+w3 +xx";
    int[] expDocNrs = {2,3};
    queriesTest(queryText, expDocNrs);
  }

  @Test
  public void testQueries02() throws Exception {
    String queryText = "+w3 xx";
    int[] expDocNrs = {2,3,1,0};<commit_msg>don't double-parse query text in this test<commit_after>package org.apache.lucene.search;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */


import java.util.Random;

import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.queryParser.ParseException;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;

import static org.junit.Assert.*;

/** Test BooleanQuery2 against BooleanQuery by overriding the standard query parser.
 * This also tests the scoring order of BooleanQuery.
 */
public class TestBoolean2 extends LuceneTestCase {
  private static IndexSearcher searcher;
  private static IndexSearcher bigSearcher;
  private static IndexReader reader;
  private static int NUM_EXTRA_DOCS = 6000;

  public static final String field = "field";
  private static Directory directory;
  private static Directory dir2;
  private static int mulFactor;

  @BeforeClass
  public static void beforeClass() throws Exception {
    directory = newDirectory();
    RandomIndexWriter writer= new RandomIndexWriter(random, directory);
    for (int i = 0; i < docFields.length; i++) {
      Document doc = new Document();
      doc.add(newField(field, docFields[i], Field.Store.NO, Field.Index.ANALYZED));
      writer.addDocument(doc);
    }
    writer.close();
    searcher = new IndexSearcher(directory, true);

    // Make big index
    dir2 = new MockDirectoryWrapper(new RAMDirectory(directory));

    // First multiply small test index:
    mulFactor = 1;
    int docCount = 0;
    do {
      final Directory copy = new MockDirectoryWrapper(new RAMDirectory(dir2));
      RandomIndexWriter w = new RandomIndexWriter(random, dir2);
      w.addIndexes(new Directory[] {copy});
      docCount = w.maxDoc();
      w.close();
      mulFactor *= 2;
    } while(docCount < 3000);

    RandomIndexWriter w = new RandomIndexWriter(random, dir2, 
        newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))
        .setMaxBufferedDocs(_TestUtil.nextInt(random, 50, 1000)));
    Document doc = new Document();
    doc.add(newField("field2", "xxx", Field.Store.NO, Field.Index.ANALYZED));
    for(int i=0;i<NUM_EXTRA_DOCS/2;i++) {
      w.addDocument(doc);
    }
    doc = new Document();
    doc.add(newField("field2", "big bad bug", Field.Store.NO, Field.Index.ANALYZED));
    for(int i=0;i<NUM_EXTRA_DOCS/2;i++) {
      w.addDocument(doc);
    }
    reader = w.getReader();
    bigSearcher = new IndexSearcher(reader);
    w.close();
  }

  @AfterClass
  public static void afterClass() throws Exception {
    searcher.close();
    reader.close();
    dir2.close();
    directory.close();
    bigSearcher.close();
    searcher = null;
    reader = null;
    dir2 = null;
    directory = null;
    bigSearcher = null;
  }

  private static String[] docFields = {
    "w1 w2 w3 w4 w5",
    "w1 w3 w2 w3",
    "w1 xx w2 yy w3",
    "w1 w3 xx w2 yy w3"
  };

  public Query makeQuery(String queryText) throws ParseException {
    Query q = (new QueryParser(TEST_VERSION_CURRENT, field, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))).parse(queryText);
    return q;
  }

  public void queriesTest(String queryText, int[] expDocNrs) throws Exception {
//System.out.println();
//System.out.println("Query: " + queryText);

    Query query = makeQuery(queryText);
    TopScoreDocCollector collector = TopScoreDocCollector.create(1000, false);
    searcher.search(query, null, collector);
    ScoreDoc[] hits1 = collector.topDocs().scoreDocs;

    collector = TopScoreDocCollector.create(1000, true);
    searcher.search(query, null, collector);
    ScoreDoc[] hits2 = collector.topDocs().scoreDocs; 

    assertEquals(mulFactor * collector.totalHits,
                 bigSearcher.search(query, 1).totalHits);
      
    CheckHits.checkHitsQuery(query, hits1, hits2, expDocNrs);
  }

  @Test
  public void testQueries01() throws Exception {
    String queryText = "+w3 +xx";
    int[] expDocNrs = {2,3};
    queriesTest(queryText, expDocNrs);
  }

  @Test
  public void testQueries02() throws Exception {
    String queryText = "+w3 xx";
    int[] expDocNrs = {2,3,1,0};
{'fraction_non_alphanumeric': 0.10568580395052612, 'fraction_numerical': 0.01495292597378623, 'mean_word_length': 3.8500447627573857, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3519721', 'n_tokens_mistral': 3371, 'n_tokens_neox': 3246, 'n_words': 922}