<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.solr.cloud;

import org.apache.commons.lang.StringUtils;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.FieldStatsInfo;
import org.apache.solr.client.solrj.response.PivotField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.StatsParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.util.DateFormatUtil;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.Arrays;
import java.util.Date;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import static org.apache.solr.common.params.FacetParams.FACET;
import static org.apache.solr.common.params.FacetParams.FACET_LIMIT;
import static org.apache.solr.common.params.FacetParams.FACET_MISSING;
import static org.apache.solr.common.params.FacetParams.FACET_OFFSET;
import static org.apache.solr.common.params.FacetParams.FACET_OVERREQUEST_COUNT;
import static org.apache.solr.common.params.FacetParams.FACET_OVERREQUEST_RATIO;
import static org.apache.solr.common.params.FacetParams.FACET_PIVOT;
import static org.apache.solr.common.params.FacetParams.FACET_PIVOT_MINCOUNT;
import static org.apache.solr.common.params.FacetParams.FACET_SORT;

/**
 * <p>
 * Randomized testing of Pivot Faceting using SolrCloud.
 * </p>
 * <p>
 * After indexing a bunch of random docs, picks some random fields to pivot facet on, 
 * and then confirms that the resulting counts match the results of filtering on those 
 * values.  This gives us strong assertions on the correctness of the total counts for 
 * each pivot value, but no assertions that the correct "top" counts were chosen.
 * </p>
 * <p>
 * NOTE: this test ignores the control collection and only deals with the 
 * CloudSolrServer - this is because the randomized field values make it very easy for 
 * the term stats to miss values even with the overrequest.
 * (because so many values will tie for "1").  What we care about here is 
 * that the counts we get back are correct and match what we get when filtering on those 
 * constraints.
 * </p>
 *
 *
 *
 */
@SuppressSSL // Too Slow
public class TestCloudPivotFacet extends AbstractFullDistribZkTestBase {

  public static Logger log = LoggerFactory.getLogger(TestCloudPivotFacet.class);

  // param used by test purely for tracing & validation
  private static String TRACE_MIN = "_test_min";
  // param used by test purely for tracing & validation
  private static String TRACE_MISS = "_test_miss";
  // param used by test purely for tracing & validation
  private static String TRACE_SORT = "_test_sort";

  /** 
   * Controls the odds of any given doc having a value in any given field -- as this gets lower, 
   * the counts for "facet.missing" pivots should increase.
   * @see #useField()
   */
  private static int useFieldRandomizedFactor = -1;

  @BeforeClass
  public static void initUseFieldRandomizedFactor() {
    useFieldRandomizedFactor = TestUtil.nextInt(random(), 2, 30);
    log.info("init'ing useFieldRandomizedFactor = {}", useFieldRandomizedFactor);
  }

  @Test
  public void test() throws Exception {

    sanityCheckAssertNumerics();

    waitForThingsToLevelOut(30000); // TODO: why whould we have to wait?
    // 
    handle.clear();
    handle.put("QTime", SKIPVAL);
    handle.put("timestamp", SKIPVAL);
    
    final Set<String> fieldNameSet = new HashSet<>();
    
    // build up a randomized index
    final int numDocs = atLeast(500);
    log.info("numDocs: {}", numDocs);

    for (int i = 1; i <= numDocs; i++) {
      SolrInputDocument doc = buildRandomDocument(i);

      // not efficient, but it guarantees that even if people change buildRandomDocument
      // we'll always have the full list of fields w/o needing to keep code in sync
      fieldNameSet.addAll(doc.getFieldNames());

      cloudClient.add(doc);
    }
    cloudClient.commit();

    fieldNameSet.remove("id");
    assertTrue("WTF, bogus field exists?", fieldNameSet.add("bogus_not_in_any_doc_s"));

    final String[] fieldNames = fieldNameSet.toArray(new String[fieldNameSet.size()]);
    Arrays.sort(fieldNames); // need determinism when picking random fields


    for (int i = 0; i < 5; i++) {

      String q = "*:*";
      if (random().nextBoolean()) {
        q = "id:[* TO " + TestUtil.nextInt(random(),300,numDocs) + "]";
      }
      ModifiableSolrParams baseP = params("rows", "0", "q", q);
      
      if (random().nextBoolean()) {
        baseP.add("fq", "id:[* TO " + TestUtil.nextInt(random(),200,numDocs) + "]");
      }

      final boolean stats = random().nextBoolean();
      if (stats) {
        baseP.add(StatsParams.STATS, "true");
        
        // if we are doing stats, then always generated the same # of STATS_FIELD
        // params, using multiple tags from a fixed set, but with diff fieldName values.
        // later, each pivot will randomly pick a tag.
        baseP.add(StatsParams.STATS_FIELD, "{!key=sk1 tag=st1,st2}" +
                  pickRandomStatsFields(fieldNames));
        baseP.add(StatsParams.STATS_FIELD, "{!key=sk2 tag=st2,st3}" +
                  pickRandomStatsFields(fieldNames));
        baseP.add(StatsParams.STATS_FIELD, "{!key=sk3 tag=st3,st4}" +
                  pickRandomStatsFields(fieldNames));
        // NOTE: there's a chance that some of those stats field names
        // will be the same, but if so, all the better to test that edge case
      }
      
      ModifiableSolrParams pivotP = params(FACET,"true");
      pivotP.add(FACET_PIVOT, buildPivotParamValue(buildRandomPivot(fieldNames)));
                 
      if (random().nextBoolean()) {
        pivotP.add(FACET_PIVOT, buildPivotParamValue(buildRandomPivot(fieldNames)));
      }

      // keep limit low - lots of unique values, and lots of depth in pivots
      pivotP.add(FACET_LIMIT, ""+TestUtil.nextInt(random(),1,17));

      // sometimes use an offset
      if (random().nextBoolean()) {
        pivotP.add(FACET_OFFSET, ""+TestUtil.nextInt(random(),0,7));
      }

      if (random().nextBoolean()) {
        String min = ""+TestUtil.nextInt(random(),0,numDocs+10);
        pivotP.add(FACET_PIVOT_MINCOUNT, min);<commit_msg>SOLR-7804: Fix test bug where it was randomly generating 2 identical facet.pivot params w/o realizing it<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.solr.cloud;

import org.apache.commons.lang.StringUtils;
import org.apache.lucene.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.response.FieldStatsInfo;
import org.apache.solr.client.solrj.response.PivotField;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.StatsParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.util.DateFormatUtil;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.util.Arrays;
import java.util.Date;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import static org.apache.solr.common.params.FacetParams.FACET;
import static org.apache.solr.common.params.FacetParams.FACET_LIMIT;
import static org.apache.solr.common.params.FacetParams.FACET_MISSING;
import static org.apache.solr.common.params.FacetParams.FACET_OFFSET;
import static org.apache.solr.common.params.FacetParams.FACET_OVERREQUEST_COUNT;
import static org.apache.solr.common.params.FacetParams.FACET_OVERREQUEST_RATIO;
import static org.apache.solr.common.params.FacetParams.FACET_PIVOT;
import static org.apache.solr.common.params.FacetParams.FACET_PIVOT_MINCOUNT;
import static org.apache.solr.common.params.FacetParams.FACET_SORT;

/**
 * <p>
 * Randomized testing of Pivot Faceting using SolrCloud.
 * </p>
 * <p>
 * After indexing a bunch of random docs, picks some random fields to pivot facet on, 
 * and then confirms that the resulting counts match the results of filtering on those 
 * values.  This gives us strong assertions on the correctness of the total counts for 
 * each pivot value, but no assertions that the correct "top" counts were chosen.
 * </p>
 * <p>
 * NOTE: this test ignores the control collection and only deals with the 
 * CloudSolrServer - this is because the randomized field values make it very easy for 
 * the term stats to miss values even with the overrequest.
 * (because so many values will tie for "1").  What we care about here is 
 * that the counts we get back are correct and match what we get when filtering on those 
 * constraints.
 * </p>
 *
 *
 *
 */
@SuppressSSL // Too Slow
public class TestCloudPivotFacet extends AbstractFullDistribZkTestBase {

  public static Logger log = LoggerFactory.getLogger(TestCloudPivotFacet.class);

  // param used by test purely for tracing & validation
  private static String TRACE_MIN = "_test_min";
  // param used by test purely for tracing & validation
  private static String TRACE_MISS = "_test_miss";
  // param used by test purely for tracing & validation
  private static String TRACE_SORT = "_test_sort";

  /** 
   * Controls the odds of any given doc having a value in any given field -- as this gets lower, 
   * the counts for "facet.missing" pivots should increase.
   * @see #useField()
   */
  private static int useFieldRandomizedFactor = -1;

  @BeforeClass
  public static void initUseFieldRandomizedFactor() {
    useFieldRandomizedFactor = TestUtil.nextInt(random(), 2, 30);
    log.info("init'ing useFieldRandomizedFactor = {}", useFieldRandomizedFactor);
  }

  @Test
  public void test() throws Exception {

    sanityCheckAssertNumerics();

    waitForThingsToLevelOut(30000); // TODO: why whould we have to wait?
    // 
    handle.clear();
    handle.put("QTime", SKIPVAL);
    handle.put("timestamp", SKIPVAL);
    
    final Set<String> fieldNameSet = new HashSet<>();
    
    // build up a randomized index
    final int numDocs = atLeast(500);
    log.info("numDocs: {}", numDocs);

    for (int i = 1; i <= numDocs; i++) {
      SolrInputDocument doc = buildRandomDocument(i);

      // not efficient, but it guarantees that even if people change buildRandomDocument
      // we'll always have the full list of fields w/o needing to keep code in sync
      fieldNameSet.addAll(doc.getFieldNames());

      cloudClient.add(doc);
    }
    cloudClient.commit();

    fieldNameSet.remove("id");
    assertTrue("WTF, bogus field exists?", fieldNameSet.add("bogus_not_in_any_doc_s"));

    final String[] fieldNames = fieldNameSet.toArray(new String[fieldNameSet.size()]);
    Arrays.sort(fieldNames); // need determinism when picking random fields


    for (int i = 0; i < 5; i++) {

      String q = "*:*";
      if (random().nextBoolean()) {
        q = "id:[* TO " + TestUtil.nextInt(random(),300,numDocs) + "]";
      }
      ModifiableSolrParams baseP = params("rows", "0", "q", q);
      
      if (random().nextBoolean()) {
        baseP.add("fq", "id:[* TO " + TestUtil.nextInt(random(),200,numDocs) + "]");
      }

      final boolean stats = random().nextBoolean();
      if (stats) {
        baseP.add(StatsParams.STATS, "true");
        
        // if we are doing stats, then always generated the same # of STATS_FIELD
        // params, using multiple tags from a fixed set, but with diff fieldName values.
        // later, each pivot will randomly pick a tag.
        baseP.add(StatsParams.STATS_FIELD, "{!key=sk1 tag=st1,st2}" +
                  pickRandomStatsFields(fieldNames));
        baseP.add(StatsParams.STATS_FIELD, "{!key=sk2 tag=st2,st3}" +
                  pickRandomStatsFields(fieldNames));
        baseP.add(StatsParams.STATS_FIELD, "{!key=sk3 tag=st3,st4}" +
                  pickRandomStatsFields(fieldNames));
        // NOTE: there's a chance that some of those stats field names
        // will be the same, but if so, all the better to test that edge case
      }
      
      ModifiableSolrParams pivotP = params(FACET,"true");

      // put our FACET_PIVOT params in a set in case we just happen to pick the same one twice
      LinkedHashSet<String> pivotParamValues = new LinkedHashSet<String>();
      pivotParamValues.add(buildPivotParamValue(buildRandomPivot(fieldNames)));
                 
      if (random().nextBoolean()) {
        pivotParamValues.add(buildPivotParamValue(buildRandomPivot(fieldNames)));
      }
      pivotP.set(FACET_PIVOT, pivotParamValues.toArray(new String[pivotParamValues.size()]));

      // keep limit low - lots of unique values, and lots of depth in pivots
      pivotP.add(FACET_LIMIT, ""+TestUtil.nextInt(random(),1,17));

      // sometimes use an offset
      if (random().nextBoolean()) {
        pivotP.add(FACET_OFFSET, ""+TestUtil.nextInt(random(),0,7));
      }

      if (random().nextBoolean()) {
        String min = ""+TestUtil.nextInt(random(),0,numDocs+10);
        pivotP.add(FACET_PIVOT_MINCOUNT, min);
{'fraction_non_alphanumeric': 0.10463515374024782, 'fraction_numerical': 0.006490526453812364, 'mean_word_length': 3.639294403892944, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2404169', 'n_tokens_mistral': 4710, 'n_tokens_neox': 4535, 'n_words': 1363}