<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.solr.update;

import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static org.junit.Assert.assertEquals;

import java.lang.invoke.MethodHandles;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;

import org.apache.lucene.util.Constants;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrEventListener;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.AbstractSolrTestCase;
import org.junit.Before;
import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Test auto commit functionality in a way that doesn't suck.
 * <p>
 * AutoCommitTest is an abomination that is way to brittle in how it 
 * tries to check that commits happened, and when they happened.
 * The goal of this test class is to (ultimately) completely replace all 
 * of the functionality of that test class using:
 * </p>
 * <ul>
 *   <li>A more robust monitor of commit/newSearcher events that records 
 *       the times of those events in a queue that can be polled.  
 *       Multiple events in rapid succession are not lost.
 *   </li>
 *   <li>Timing checks that are forgiving of slow machines and use 
 *       knowledge of how slow A-&gt;B was to affect the expectation of 
 *       how slow B-&gt;C will be
 *   </li>
 * </ul>
 */
@Slow
public class SoftAutoCommitTest extends AbstractSolrTestCase {
  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

  @BeforeClass
  public static void beforeClass() throws Exception {
    initCore("solrconfig.xml", "schema.xml");
  }

  private MockEventListener monitor;
  private DirectUpdateHandler2 updater;
    
  @Before
  public void createMonitor() throws Exception {
    assumeFalse("This test is not working on Windows (or maybe machines with only 2 CPUs)",
      Constants.WINDOWS);
  
    SolrCore core = h.getCore();

    updater = (DirectUpdateHandler2) core.getUpdateHandler();
    monitor = new MockEventListener();

    core.registerNewSearcherListener(monitor);
    updater.registerSoftCommitCallback(monitor);
    updater.registerCommitCallback(monitor);

    // isolate searcher getting ready from this test
    monitor.searcher.poll(5000, MILLISECONDS);
  }
  
  @Override
  public void setUp() throws Exception {
    super.setUp();

  }
  
  public void testSoftAndHardCommitMaxDocs() throws Exception {

    // NOTE WHEN READING THIS TEST...
    // The maxDocs settings on the CommitTrackers are the "upper bound"
    // of how many docs can be added with out doing a commit.
    // That means they are one less then the actual number of docs that will trigger a commit.
    final int softCommitMaxDocs = 5;
    final int hardCommitMaxDocs = 7;

    assert softCommitMaxDocs < hardCommitMaxDocs; // remainder of test designed with these assumptions
    
    CommitTracker hardTracker = updater.commitTracker;
    CommitTracker softTracker = updater.softCommitTracker;
    
    // wait out any leaked commits
    monitor.hard.poll(3000, MILLISECONDS);
    monitor.soft.poll(0, MILLISECONDS);
    monitor.clear();
    
    softTracker.setDocsUpperBound(softCommitMaxDocs);
    softTracker.setTimeUpperBound(-1);
    hardTracker.setDocsUpperBound(hardCommitMaxDocs);
    hardTracker.setTimeUpperBound(-1);
    // simplify whats going on by only having soft auto commits trigger new searchers
    hardTracker.setOpenSearcher(false);

    // Note: doc id counting starts at 0, see comment at start of test regarding "upper bound"

    // add num docs up to the soft commit upper bound
    for (int i = 0; i < softCommitMaxDocs; i++) {
      assertU(adoc("id", ""+(8000 + i), "subject", "testMaxDocs"));
    }
    // the first soft commit we see must be after this.
    final long minSoftCommitNanos = System.nanoTime();
    
    // now add the doc that will trigger the soft commit,
    // as well as additional docs up to the hard commit upper bound
    for (int i = softCommitMaxDocs; i < hardCommitMaxDocs; i++) {
      assertU(adoc("id", ""+(8000 + i), "subject", "testMaxDocs"));
    }
    // the first hard commit we see must be after this.
    final long minHardCommitNanos = System.nanoTime();

    // a final doc to trigger the hard commit
    assertU(adoc("id", ""+(8000 + hardCommitMaxDocs), "subject", "testMaxDocs"));

    // now poll our monitors for the timestamps on the first commits
    final Long firstSoftNanos = monitor.soft.poll(5000, MILLISECONDS);
    final Long firstHardNanos = monitor.hard.poll(5000, MILLISECONDS);

    assertNotNull("didn't get a single softCommit after adding the max docs", firstSoftNanos);
    assertNotNull("didn't get a single hardCommit after adding the max docs", firstHardNanos);
                  
    assertTrue("softCommit @ " + firstSoftNanos + "ns is before the maxDocs should have triggered it @ " +
               minSoftCommitNanos + "ns",
               minSoftCommitNanos < firstSoftNanos);
    assertTrue("hardCommit @ " + firstHardNanos + "ns is before the maxDocs should have triggered it @ " +
               minHardCommitNanos + "ns",
               minHardCommitNanos < firstHardNanos);

    final Long firstSearcherNanos = monitor.searcher.poll(5000, MILLISECONDS);
    assertNotNull("didn't get a single new searcher", firstSearcherNanos);
    for (int i = 0; i <= softCommitMaxDocs; i++) {
      assertQ("should find one", req("id:"+(8000 + i)) ,"//result[@numFound=1]" );
    }

    // wait a bit, w/o other action we shouldn't see any new hard/soft commits 
    assertNull("Got a hard commit we weren't expecting",
               monitor.hard.poll(1000, MILLISECONDS));
    assertNull("Got a soft commit we weren't expecting",
               monitor.soft.poll(0, MILLISECONDS));
    
    monitor.assertSaneOffers();
    monitor.clear();
  }

  public void testSoftAndHardCommitMaxTimeMixedAdds() throws Exception {
    final int softCommitWaitMillis = 500;
    final int hardCommitWaitMillis = 1200;

    CommitTracker hardTracker = updater.commitTracker;
    CommitTracker softTracker = updater.softCommitTracker;
    
    // wait out any leaked commits
    monitor.soft.poll(softCommitWaitMillis * 2, MILLISECONDS);
    monitor.hard.poll(hardCommitWaitMillis * 2, MILLISECONDS);
    <commit_msg>SOLR-11054: Revert the last commit since we found a test failure<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.apache.solr.update;

import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
import static org.junit.Assert.assertEquals;

import java.lang.invoke.MethodHandles;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;

import org.apache.lucene.util.Constants;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrEventListener;
import org.apache.solr.search.SolrIndexSearcher;
import org.apache.solr.util.AbstractSolrTestCase;
import org.junit.Before;
import org.junit.BeforeClass;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Test auto commit functionality in a way that doesn't suck.
 * <p>
 * AutoCommitTest is an abomination that is way to brittle in how it 
 * tries to check that commits happened, and when they happened.
 * The goal of this test class is to (ultimately) completely replace all 
 * of the functionality of that test class using:
 * </p>
 * <ul>
 *   <li>A more robust monitor of commit/newSearcher events that records 
 *       the times of those events in a queue that can be polled.  
 *       Multiple events in rapid succession are not lost.
 *   </li>
 *   <li>Timing checks that are forgiving of slow machines and use 
 *       knowledge of how slow A-&gt;B was to affect the expectation of 
 *       how slow B-&gt;C will be
 *   </li>
 * </ul>
 */
@Slow
public class SoftAutoCommitTest extends AbstractSolrTestCase {
  private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());

  @BeforeClass
  public static void beforeClass() throws Exception {
    initCore("solrconfig.xml", "schema.xml");
  }

  private MockEventListener monitor;
  private DirectUpdateHandler2 updater;
    
  @Before
  public void createMonitor() throws Exception {
    assumeFalse("This test is not working on Windows (or maybe machines with only 2 CPUs)",
      Constants.WINDOWS);
  
    SolrCore core = h.getCore();

    updater = (DirectUpdateHandler2) core.getUpdateHandler();
    monitor = new MockEventListener();

    core.registerNewSearcherListener(monitor);
    updater.registerSoftCommitCallback(monitor);
    updater.registerCommitCallback(monitor);

    // isolate searcher getting ready from this test
    monitor.searcher.poll(5000, MILLISECONDS);
  }
  
  @Override
  public void setUp() throws Exception {
    super.setUp();

  }
  
  public void testSoftAndHardCommitMaxDocs() throws Exception {

    // NOTE WHEN READING THIS TEST...
    // The maxDocs settings on the CommitTrackers are the "upper bound"
    // of how many docs can be added with out doing a commit.
    // That means they are one less then the actual number of docs that will trigger a commit.
    final int softCommitMaxDocs = 5;
    final int hardCommitMaxDocs = 7;

    assert softCommitMaxDocs < hardCommitMaxDocs; // remainder of test designed with these assumptions
    
    CommitTracker hardTracker = updater.commitTracker;
    CommitTracker softTracker = updater.softCommitTracker;
    
    // wait out any leaked commits
    monitor.hard.poll(3000, MILLISECONDS);
    monitor.soft.poll(0, MILLISECONDS);
    monitor.clear();
    
    softTracker.setDocsUpperBound(softCommitMaxDocs);
    softTracker.setTimeUpperBound(-1);
    hardTracker.setDocsUpperBound(hardCommitMaxDocs);
    hardTracker.setTimeUpperBound(-1);
    // simplify whats going on by only having soft auto commits trigger new searchers
    hardTracker.setOpenSearcher(false);

    // Note: doc id counting starts at 0, see comment at start of test regarding "upper bound"

    // add num docs up to the soft commit upper bound
    for (int i = 0; i < softCommitMaxDocs; i++) {
      assertU(adoc("id", ""+(8000 + i), "subject", "testMaxDocs"));
    }
    // the first soft commit we see must be after this.
    final long minSoftCommitNanos = System.nanoTime();
    
    // now add the doc that will trigger the soft commit,
    // as well as additional docs up to the hard commit upper bound
    for (int i = softCommitMaxDocs; i < hardCommitMaxDocs; i++) {
      assertU(adoc("id", ""+(8000 + i), "subject", "testMaxDocs"));
    }
    // the first hard commit we see must be after this.
    final long minHardCommitNanos = System.nanoTime();

    // a final doc to trigger the hard commit
    assertU(adoc("id", ""+(8000 + hardCommitMaxDocs), "subject", "testMaxDocs"));

    // now poll our monitors for the timestamps on the first commits
    final Long firstSoftNanos = monitor.soft.poll(5000, MILLISECONDS);
    final Long firstHardNanos = monitor.hard.poll(5000, MILLISECONDS);

    assertNotNull("didn't get a single softCommit after adding the max docs", firstSoftNanos);
    assertNotNull("didn't get a single hardCommit after adding the max docs", firstHardNanos);
                  
    assertTrue("softCommit @ " + firstSoftNanos + "ns is before the maxDocs should have triggered it @ " +
               minSoftCommitNanos + "ns",
               minSoftCommitNanos < firstSoftNanos);
    assertTrue("hardCommit @ " + firstHardNanos + "ns is before the maxDocs should have triggered it @ " +
               minHardCommitNanos + "ns",
               minHardCommitNanos < firstHardNanos);

    // wait a bit, w/o other action we shouldn't see any new hard/soft commits 
    assertNull("Got a hard commit we weren't expecting",
               monitor.hard.poll(1000, MILLISECONDS));
    assertNull("Got a soft commit we weren't expecting",
               monitor.soft.poll(0, MILLISECONDS));
    
    monitor.assertSaneOffers();
    monitor.clear();
  }

  public void testSoftAndHardCommitMaxTimeMixedAdds() throws Exception {
    final int softCommitWaitMillis = 500;
    final int hardCommitWaitMillis = 1200;

    CommitTracker hardTracker = updater.commitTracker;
    CommitTracker softTracker = updater.softCommitTracker;
    
    // wait out any leaked commits
    monitor.soft.poll(softCommitWaitMillis * 2, MILLISECONDS);
    monitor.hard.poll(hardCommitWaitMillis * 2, MILLISECONDS);
    
{'fraction_non_alphanumeric': 0.08635542587189106, 'fraction_numerical': 0.009101014311518689, 'mean_word_length': 3.559708584098828, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '138357', 'n_tokens_mistral': 4216, 'n_tokens_neox': 4095, 'n_words': 1421}