<commit_before>package org.apache.lucene.index;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.IOException;

import org.apache.lucene.store.Directory;
import org.apache.lucene.codecs.PerDocProducer;
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader;
import org.apache.lucene.codecs.lucene40.BitVector; // nocommit: move asserts/checks to codec
import org.apache.lucene.search.FieldCache; // javadocs
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.MutableBits;

/**
 * @lucene.experimental
 */
public final class SegmentReader extends IndexReader {

  private final SegmentInfo si;
  private final ReaderContext readerContext = new AtomicReaderContext(this);
  
  private final Bits liveDocs;

  // Normally set to si.docCount - si.delDocCount, unless we
  // were created as an NRT reader from IW, in which case IW
  // tells us the docCount:
  private final int numDocs;

  private final SegmentCoreReaders core;

  /**
   * @throws CorruptIndexException if the index is corrupt
   * @throws IOException if there is a low-level IO error
   */
  public SegmentReader(SegmentInfo si, int termInfosIndexDivisor, IOContext context) throws IOException {
    this.si = si;
    core = new SegmentCoreReaders(this, si.dir, si, context, termInfosIndexDivisor);
    boolean success = false;
    try {
      if (si.hasDeletions()) {
        // NOTE: the bitvector is stored using the regular directory, not cfs
        liveDocs = si.getCodec().liveDocsFormat().readLiveDocs(directory(), si, new IOContext(IOContext.READ, true));
      } else {
        assert si.getDelCount() == 0;
        liveDocs = null;
      }
      numDocs = si.docCount - si.getDelCount();
      assert checkLiveCounts(false);
      success = true;
    } finally {
      // With lock-less commits, it's entirely possible (and
      // fine) to hit a FileNotFound exception above.  In
      // this case, we want to explicitly close any subset
      // of things that were opened so that we don't have to
      // wait for a GC to do so.
      if (!success) {
        core.decRef();
      }
    }
  }

  // TODO: really these next 2 ctors could take
  // SegmentCoreReaders... that's all we do w/ the parent
  // SR:

  // Create new SegmentReader sharing core from a previous
  // SegmentReader and loading new live docs from a new
  // deletes file.  Used by openIfChanged.
  SegmentReader(SegmentInfo si, SegmentReader parent, IOContext context) throws IOException {
    assert si.dir == parent.getSegmentInfo().dir;
    this.si = si;

    // It's no longer possible to unDeleteAll, so, we can
    // only be created if we have deletions:
    assert si.hasDeletions();

    // ... but load our own deleted docs:
    liveDocs = si.getCodec().liveDocsFormat().readLiveDocs(si.dir, si, context);
    numDocs = si.docCount - si.getDelCount();
    assert checkLiveCounts(false);

    // We share core w/ parent:
    parent.core.incRef();
    core = parent.core;
  }

  // Create new SegmentReader sharing core from a previous
  // SegmentReader and using the provided in-memory
  // liveDocs.  Used by IndexWriter to provide a new NRT
  // reader:
  SegmentReader(SegmentReader parent, MutableBits liveDocs, int numDocs) throws IOException {
    this.si = parent.si;
    parent.core.incRef();
    this.core = parent.core;

    assert liveDocs != null;
    this.liveDocs = liveDocs;

    this.numDocs = numDocs;

    assert checkLiveCounts(true);
  }

  @Override
  public Bits getLiveDocs() {
    ensureOpen();
    return liveDocs;
  }

  // nocommit
  private boolean checkLiveCounts(boolean isNRT) throws IOException {
    BitVector liveDocs = (BitVector) this.liveDocs;
    if (liveDocs != null) {
      if (liveDocs.size() != si.docCount) {
        throw new CorruptIndexException("document count mismatch: deleted docs count " + liveDocs.size() + " vs segment doc count " + si.docCount + " segment=" + si.name);
      }

      final int recomputedCount = liveDocs.getRecomputedCount();
      // Verify BitVector is self consistent:
      assert liveDocs.count() == recomputedCount : "live count=" + liveDocs.count() + " vs recomputed count=" + recomputedCount;

      // Verify our docCount matches:
      assert numDocs == recomputedCount :
      "delete count mismatch: numDocs=" + numDocs + " vs BitVector=" + (si.docCount-recomputedCount);

      assert isNRT || si.docCount - si.getDelCount() == recomputedCount :
        "si.docCount=" + si.docCount + "si.getDelCount()=" + si.getDelCount() + " recomputedCount=" + recomputedCount;
    }
  
    return true;<commit_msg>LUCENE-3661: remove bitvector from segmentreader<commit_after>package org.apache.lucene.index;

/**
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.IOException;

import org.apache.lucene.store.Directory;
import org.apache.lucene.codecs.PerDocProducer;
import org.apache.lucene.codecs.StoredFieldsReader;
import org.apache.lucene.codecs.TermVectorsReader;
import org.apache.lucene.search.FieldCache; // javadocs
import org.apache.lucene.store.IOContext;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.MutableBits;

/**
 * @lucene.experimental
 */
public final class SegmentReader extends IndexReader {

  private final SegmentInfo si;
  private final ReaderContext readerContext = new AtomicReaderContext(this);
  
  private final Bits liveDocs;

  // Normally set to si.docCount - si.delDocCount, unless we
  // were created as an NRT reader from IW, in which case IW
  // tells us the docCount:
  private final int numDocs;

  private final SegmentCoreReaders core;

  /**
   * @throws CorruptIndexException if the index is corrupt
   * @throws IOException if there is a low-level IO error
   */
  public SegmentReader(SegmentInfo si, int termInfosIndexDivisor, IOContext context) throws IOException {
    this.si = si;
    core = new SegmentCoreReaders(this, si.dir, si, context, termInfosIndexDivisor);
    boolean success = false;
    try {
      if (si.hasDeletions()) {
        // NOTE: the bitvector is stored using the regular directory, not cfs
        liveDocs = si.getCodec().liveDocsFormat().readLiveDocs(directory(), si, new IOContext(IOContext.READ, true));
      } else {
        assert si.getDelCount() == 0;
        liveDocs = null;
      }
      numDocs = si.docCount - si.getDelCount();
      assert checkLiveCounts(false);
      success = true;
    } finally {
      // With lock-less commits, it's entirely possible (and
      // fine) to hit a FileNotFound exception above.  In
      // this case, we want to explicitly close any subset
      // of things that were opened so that we don't have to
      // wait for a GC to do so.
      if (!success) {
        core.decRef();
      }
    }
  }

  // TODO: really these next 2 ctors could take
  // SegmentCoreReaders... that's all we do w/ the parent
  // SR:

  // Create new SegmentReader sharing core from a previous
  // SegmentReader and loading new live docs from a new
  // deletes file.  Used by openIfChanged.
  SegmentReader(SegmentInfo si, SegmentReader parent, IOContext context) throws IOException {
    assert si.dir == parent.getSegmentInfo().dir;
    this.si = si;

    // It's no longer possible to unDeleteAll, so, we can
    // only be created if we have deletions:
    assert si.hasDeletions();

    // ... but load our own deleted docs:
    liveDocs = si.getCodec().liveDocsFormat().readLiveDocs(si.dir, si, context);
    numDocs = si.docCount - si.getDelCount();
    assert checkLiveCounts(false);

    // We share core w/ parent:
    parent.core.incRef();
    core = parent.core;
  }

  // Create new SegmentReader sharing core from a previous
  // SegmentReader and using the provided in-memory
  // liveDocs.  Used by IndexWriter to provide a new NRT
  // reader:
  SegmentReader(SegmentReader parent, MutableBits liveDocs, int numDocs) throws IOException {
    this.si = parent.si;
    parent.core.incRef();
    this.core = parent.core;

    assert liveDocs != null;
    this.liveDocs = liveDocs;

    this.numDocs = numDocs;

    assert checkLiveCounts(true);
  }

  @Override
  public Bits getLiveDocs() {
    ensureOpen();
    return liveDocs;
  }

  private boolean checkLiveCounts(boolean isNRT) throws IOException {
    MutableBits liveDocs = (MutableBits) this.liveDocs;
    if (liveDocs != null) {
      if (liveDocs.length() != si.docCount) {
        throw new CorruptIndexException("document count mismatch: deleted docs count " + liveDocs.length() + " vs segment doc count " + si.docCount + " segment=" + si.name);
      }

      final int count = liveDocs.count();

      // Verify our docCount matches:
      assert numDocs == count :
      "delete count mismatch: numDocs=" + numDocs + " vs BitVector=" + (si.docCount-count);

      assert isNRT || si.docCount - si.getDelCount() == count :
        "si.docCount=" + si.docCount + "si.getDelCount()=" + si.getDelCount() + " recomputedCount=" + count;
    }
  
    return true;
{'fraction_non_alphanumeric': 0.09631244667741018, 'fraction_numerical': 0.001706322874206086, 'mean_word_length': 3.1666666666666665, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5448930', 'n_tokens_mistral': 3173, 'n_tokens_neox': 3050, 'n_words': 1048}