<commit_before>// Copyright (C) 2013 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.gerrit.lucene;

import static java.util.concurrent.TimeUnit.MILLISECONDS;

import com.google.common.base.Joiner;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.AbstractFuture;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.FieldDef;
import com.google.gerrit.server.index.FieldDef.FillArgs;
import com.google.gerrit.server.index.FieldType;
import com.google.gerrit.server.index.Index;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.index.Schema.Values;

import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TrackingIndexWriter;
import org.apache.lucene.search.ControlledRealTimeReopenThread;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ReferenceManager;
import org.apache.lucene.search.ReferenceManager.RefreshListener;
import org.apache.lucene.search.SearcherFactory;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.sql.Timestamp;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;

/** Basic Lucene index implementation. */
public abstract class AbstractLuceneIndex<K, V> implements Index<K, V> {
  private static final Logger log =
      LoggerFactory.getLogger(AbstractLuceneIndex.class);

  static String sortFieldName(FieldDef<?, ?> f) {
    return f.getName() + "_SORT";
  }

  public static void setReady(SitePaths sitePaths, String name, int version,
      boolean ready) throws IOException {
    try {
      GerritIndexStatus cfg = new GerritIndexStatus(sitePaths);
      cfg.setReady(name, version, ready);
      cfg.save();
    } catch (ConfigInvalidException e) {
      throw new IOException(e);
    }
  }

  private final Schema<V> schema;
  private final SitePaths sitePaths;
  private final Directory dir;
  private final String name;
  private final TrackingIndexWriter writer;
  private final ReferenceManager<IndexSearcher> searcherManager;
  private final ControlledRealTimeReopenThread<IndexSearcher> reopenThread;
  private final Set<NrtFuture> notDoneNrtFutures;

  AbstractLuceneIndex(
      Schema<V> schema,
      SitePaths sitePaths,
      Directory dir,
      String name,
      String subIndex,
      GerritIndexWriterConfig writerConfig,
      SearcherFactory searcherFactory) throws IOException {
    this.schema = schema;
    this.sitePaths = sitePaths;
    this.dir = dir;
    this.name = name;
    final String index = Joiner.on('_').skipNulls().join(name, subIndex);
    IndexWriter delegateWriter;
    long commitPeriod = writerConfig.getCommitWithinMs();

    if (commitPeriod < 0) {
      delegateWriter = new AutoCommitWriter(dir, writerConfig.getLuceneConfig());
    } else if (commitPeriod == 0) {
      delegateWriter =
          new AutoCommitWriter(dir, writerConfig.getLuceneConfig(), true);
    } else {
      final AutoCommitWriter autoCommitWriter =
          new AutoCommitWriter(dir, writerConfig.getLuceneConfig());
      delegateWriter = autoCommitWriter;

      new ScheduledThreadPoolExecutor(1, new ThreadFactoryBuilder()
          .setNameFormat("Commit-%d " + index)
          .setDaemon(true)
          .build())
          .scheduleAtFixedRate(new Runnable() {
            @Override
            public void run() {
              try {
                if (autoCommitWriter.hasUncommittedChanges()) {
                  autoCommitWriter.manualFlush();
                  autoCommitWriter.commit();
                }
              } catch (IOException e) {
                log.error("Error committing " + index + " Lucene index", e);
              } catch (OutOfMemoryError e) {
                log.error("Error committing " + index + " Lucene index", e);
                try {
                  autoCommitWriter.close();
                } catch (IOException e2) {
                  log.error("SEVERE: Error closing " + index
                      + " Lucene index  after OOM; index may be corrupted.", e);
                }
              }
            }
          }, commitPeriod, commitPeriod, MILLISECONDS);
    }
    writer = new TrackingIndexWriter(delegateWriter);
    searcherManager = new WrappableSearcherManager(
        writer.getIndexWriter(), true, searcherFactory);

    notDoneNrtFutures = Sets.newConcurrentHashSet();

    reopenThread = new ControlledRealTimeReopenThread<>(
        writer, searcherManager,
        0.500 /* maximum stale age (seconds) */,
        0.010 /* minimum stale age (seconds) */);
    reopenThread.setName("NRT " + name);
    reopenThread.setPriority(Math.min(
        Thread.currentThread().getPriority() + 2,
        Thread.MAX_PRIORITY));
    reopenThread.setDaemon(true);

    // This must be added after the reopen thread is created. The reopen thread
    // adds its own listener which copies its internally last-refreshed
    // generation to the searching generation. removeIfDone() depends on the
    // searching generation being up to date when calling
    // reopenThread.waitForGeneration(gen, 0), therefore the reopen thread's
    // internal listener needs to be called first.
    // TODO(dborowitz): This may have been fixed by
    // http://issues.apache.org/jira/browse/LUCENE-5461
    searcherManager.addListener(new RefreshListener() {
      @Override
      public void beforeRefresh() throws IOException {
      }

      @Override
      public void afterRefresh(boolean didRefresh) throws IOException {
        for (NrtFuture f : notDoneNrtFutures) {
          f.removeIfDone();
        }
      }
    });

    reopenThread.start();
  }

  @Override
  public void markReady(boolean ready) throws IOException {
    setReady(sitePaths, name, schema.getVersion(), ready);
  }

  @Override
  public void close() {
    reopenThread.close();

    // Closing the reopen thread sets its generation to Long.MAX_VALUE, but we
    // still need to refresh the searcher manager to let pending NrtFutures
    // know.
    //
    // Any futures created after this method (which may happen due to undefined
    // shutdown ordering behavior) will finish immediately, even though they may
    // not have flushed.
    try {
      searcherManager.maybeRefreshBlocking();<commit_msg>Shutdown executor for auto-commiting Lucene index when stopping Gerrit<commit_after>// Copyright (C) 2013 The Android Open Source Project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.gerrit.lucene;

import static java.util.concurrent.TimeUnit.MILLISECONDS;

import com.google.common.base.Joiner;
import com.google.common.collect.Sets;
import com.google.common.util.concurrent.AbstractFuture;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.gerrit.server.config.SitePaths;
import com.google.gerrit.server.index.FieldDef;
import com.google.gerrit.server.index.FieldDef.FillArgs;
import com.google.gerrit.server.index.FieldType;
import com.google.gerrit.server.index.Index;
import com.google.gerrit.server.index.Schema;
import com.google.gerrit.server.index.Schema.Values;

import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TrackingIndexWriter;
import org.apache.lucene.search.ControlledRealTimeReopenThread;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ReferenceManager;
import org.apache.lucene.search.ReferenceManager.RefreshListener;
import org.apache.lucene.search.SearcherFactory;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.eclipse.jgit.errors.ConfigInvalidException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;
import java.sql.Timestamp;
import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;

/** Basic Lucene index implementation. */
public abstract class AbstractLuceneIndex<K, V> implements Index<K, V> {
  private static final Logger log =
      LoggerFactory.getLogger(AbstractLuceneIndex.class);

  static String sortFieldName(FieldDef<?, ?> f) {
    return f.getName() + "_SORT";
  }

  public static void setReady(SitePaths sitePaths, String name, int version,
      boolean ready) throws IOException {
    try {
      GerritIndexStatus cfg = new GerritIndexStatus(sitePaths);
      cfg.setReady(name, version, ready);
      cfg.save();
    } catch (ConfigInvalidException e) {
      throw new IOException(e);
    }
  }

  private final Schema<V> schema;
  private final SitePaths sitePaths;
  private final Directory dir;
  private final String name;
  private final TrackingIndexWriter writer;
  private final ReferenceManager<IndexSearcher> searcherManager;
  private final ControlledRealTimeReopenThread<IndexSearcher> reopenThread;
  private final Set<NrtFuture> notDoneNrtFutures;
  private ScheduledThreadPoolExecutor autoCommitExecutor;

  AbstractLuceneIndex(
      Schema<V> schema,
      SitePaths sitePaths,
      Directory dir,
      String name,
      String subIndex,
      GerritIndexWriterConfig writerConfig,
      SearcherFactory searcherFactory) throws IOException {
    this.schema = schema;
    this.sitePaths = sitePaths;
    this.dir = dir;
    this.name = name;
    final String index = Joiner.on('_').skipNulls().join(name, subIndex);
    IndexWriter delegateWriter;
    long commitPeriod = writerConfig.getCommitWithinMs();

    if (commitPeriod < 0) {
      delegateWriter = new AutoCommitWriter(dir, writerConfig.getLuceneConfig());
    } else if (commitPeriod == 0) {
      delegateWriter =
          new AutoCommitWriter(dir, writerConfig.getLuceneConfig(), true);
    } else {
      final AutoCommitWriter autoCommitWriter =
          new AutoCommitWriter(dir, writerConfig.getLuceneConfig());
      delegateWriter = autoCommitWriter;

      autoCommitExecutor = new ScheduledThreadPoolExecutor(1, new ThreadFactoryBuilder()
          .setNameFormat("Commit-%d " + index)
          .setDaemon(true)
          .build());
      autoCommitExecutor.scheduleAtFixedRate(new Runnable() {
            @Override
            public void run() {
              try {
                if (autoCommitWriter.hasUncommittedChanges()) {
                  autoCommitWriter.manualFlush();
                  autoCommitWriter.commit();
                }
              } catch (IOException e) {
                log.error("Error committing " + index + " Lucene index", e);
              } catch (OutOfMemoryError e) {
                log.error("Error committing " + index + " Lucene index", e);
                try {
                  autoCommitWriter.close();
                } catch (IOException e2) {
                  log.error("SEVERE: Error closing " + index
                      + " Lucene index  after OOM; index may be corrupted.", e);
                }
              }
            }
          }, commitPeriod, commitPeriod, MILLISECONDS);
    }
    writer = new TrackingIndexWriter(delegateWriter);
    searcherManager = new WrappableSearcherManager(
        writer.getIndexWriter(), true, searcherFactory);

    notDoneNrtFutures = Sets.newConcurrentHashSet();

    reopenThread = new ControlledRealTimeReopenThread<>(
        writer, searcherManager,
        0.500 /* maximum stale age (seconds) */,
        0.010 /* minimum stale age (seconds) */);
    reopenThread.setName("NRT " + name);
    reopenThread.setPriority(Math.min(
        Thread.currentThread().getPriority() + 2,
        Thread.MAX_PRIORITY));
    reopenThread.setDaemon(true);

    // This must be added after the reopen thread is created. The reopen thread
    // adds its own listener which copies its internally last-refreshed
    // generation to the searching generation. removeIfDone() depends on the
    // searching generation being up to date when calling
    // reopenThread.waitForGeneration(gen, 0), therefore the reopen thread's
    // internal listener needs to be called first.
    // TODO(dborowitz): This may have been fixed by
    // http://issues.apache.org/jira/browse/LUCENE-5461
    searcherManager.addListener(new RefreshListener() {
      @Override
      public void beforeRefresh() throws IOException {
      }

      @Override
      public void afterRefresh(boolean didRefresh) throws IOException {
        for (NrtFuture f : notDoneNrtFutures) {
          f.removeIfDone();
        }
      }
    });

    reopenThread.start();
  }

  @Override
  public void markReady(boolean ready) throws IOException {
    setReady(sitePaths, name, schema.getVersion(), ready);
  }

  @Override
  public void close() {
    if (autoCommitExecutor != null) {
      autoCommitExecutor.shutdown();
    }

    reopenThread.close();

    // Closing the reopen thread sets its generation to Long.MAX_VALUE, but we
    // still need to refresh the searcher manager to let pending NrtFutures
    // know.
    //
    // Any futures created after this method (which may happen due to undefined
    // shutdown ordering behavior) will finish immediately, even though they may
    // not have flushed.
    try {
      searcherManager.maybeRefreshBlocking();
{'fraction_non_alphanumeric': 0.09372802250495492, 'fraction_numerical': 0.00358033373825203, 'mean_word_length': 3.292535675082327, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5014863', 'n_tokens_mistral': 4400, 'n_tokens_neox': 4264, 'n_words': 1129}