<commit_before>require 'monitor'

module Ferret::Index
  # IndexReader is an abstract class, providing an interface for accessing an
  # index.  Search of an index is done entirely through this abstract interface,
  # class which implements it is searchable.
  #
  # Concrete subclasses of IndexReader are usually constructed with a call to
  # one of the static <tt>open()</tt> methods, e.g. <tt>#open</tt>.
  #
  # For efficiency, in this API documents are often referred to via
  # _document numbers_, non-negative integers which each name a unique
  # document in the index.  These document numbers are ephemeral, ie they may change
  # as documents are added to and deleted from an index.  Clients should thus not
  # rely on a given document having the same number between sessions.
  # 
  # An IndexReader can be opened on a directory for which an IndexWriter is
  # opened already, but it cannot be used to delete documents from the index then.
  class IndexReader 
    include MonitorMixin
    
    # This array contains all filename extensions used by Lucene's index files, with
    # one exception, namely the extension made up from +.f+ + a number.
    # Also note that two of Lucene's files (+deletable+ and
    # +segments+) don't have any filename extension.
    FILENAME_EXTENSIONS = ["cfs",
                           "fnm",
                           "fdx",
                           "fdt",
                           "tii",
                           "tis",
                           "frq",
                           "prx",
                           "del",
                           "tvx",
                           "tvd",
                           "tvf",
                           "tvp"]

    attr_reader :directory
    
    class FieldOption < Ferret::Utils::Parameter
      # all fields
      ALL = FieldOption.new("ALL")
      # all indexed fields
      INDEXED = FieldOption.new("INDEXED")
      # all fields which are not indexed
      UNINDEXED = FieldOption.new("UNINDEXED")
      # all fields which are indexed with termvectors enables
      INDEXED_WITH_TERM_VECTOR = FieldOption.new("INDEXED_WITH_TERM_VECTOR")
      # all fields which are indexed but don't have termvectors enabled
      INDEXED_NO_TERM_VECTOR = FieldOption.new("INDEXED_NO_TERM_VECTOR")
      # all fields where termvectors are enabled. Please note that only standard
      # termvector fields are returned
      TERM_VECTOR = FieldOption.new("TERM_VECTOR")
      # all field with termvectors wiht positions enabled
      TERM_VECTOR_WITH_POSITION = FieldOption.new("TERM_VECTOR_WITH_POSITION")
      # all fields where termvectors with offset position are set
      TERM_VECTOR_WITH_OFFSET = FieldOption.new("TERM_VECTOR_WITH_OFFSET")
      # all fields where termvectors with offset and position values set
      TERM_VECTOR_WITH_POSITION_OFFSET =
        FieldOption.new("TERM_VECTOR_WITH_POSITION_OFFSET")
    end
    
    # To create an IndexReader use the IndexReader.open method. This method
    # should only be used by subclasses.
    #
    # directory::       Directory where IndexReader files reside.
    # segment_infos::   Used for write-l
    # close_directory:: close the directory when the index reader is closed
    def initialize(directory, segment_infos = nil,
                   close_directory = false, directory_owner = false)
      super()
      @directory = directory
      @close_directory = close_directory
      @segment_infos = segment_infos
      @directory_owner = directory_owner

      @has_changes = false
      @stale = false
      @write_lock = nil

      #ObjectSpace.define_finalizer(self, lambda { |id| @write_lock.release() if @write_lock})
    end

    # Returns an index reader to read the index in the directory
    #
    # directory::       This can either be a Directory object or you can pass
    #                   nil (RamDirectory is created) or a path (FSDirectory
    #                   is created). If you chose the second or third options,
    #                   you should leave close_directory as true and infos as
    #                   nil.
    # close_directory:: True if you want the IndexReader to close the
    #                   directory when the IndexReader is closed. You'll want
    #                   to set this to false if other objects are using the
    #                   same directory object.
    # infos::           Expert: This can be used to read an different version
    #                   of the index but should really be left alone.
    def IndexReader.open(directory, close_directory = true, infos = nil)
      if directory.nil?
        directory = Ferret::Store::RAMDirectory.new
      elsif directory.is_a?(String)
        directory = Ferret::Store::FSDirectory.new(directory, true)
      end
      directory.synchronize do # in- & inter-process sync
        commit_lock = directory.make_lock(IndexWriter::COMMIT_LOCK_NAME)
        commit_lock.while_locked() do
          if infos.nil?
            infos = SegmentInfos.new()
            infos.read(directory)
          end
          if (infos.size() == 1) # index is optimized
            return SegmentReader.get(infos[0], infos, close_directory)
          end
          readers = Array.new(infos.size)
          infos.size.times do |i|
            readers[i] = SegmentReader.get(infos[i])
          end
          return MultiReader.new(readers, directory, infos, close_directory)
        end
      end
    end

    # Reads version number from segments files. The version number counts the
    # number of changes of the index.
    # 
    # directory:: where the index resides.
    # returns:: version number.
    # raises:: IOError if segments file cannot be read.
    def IndexReader.get_current_version(directory)
      return SegmentInfos.read_current_version(directory)
    end

    # Return an array of term vectors for the specified document.  The array
    # contains a vector for each vectorized field in the document.  Each vector
    # contains terms and frequencies for all terms in a given vectorized field.
    # If no such fields existed, the method returns nil. The term vectors that
    # are returned my either be of type TermFreqVector or of type
    # TermDocPosEnumVector if positions or offsets have been stored.
    # 
    # doc_number:: document for which term vectors are returned
    # returns:: array of term vectors. May be nil if no term vectors have been
    #           stored for the specified document.
    # raises:: IOError if index cannot be accessed
    #
    # See Field.TermVector
    def get_term_vectors(doc_number)
      raise NotImplementedError
    end
           

    
    # Return a term vector for the specified document and field. The returned
    # vector contains terms and frequencies for the terms in the specified
    # field of this document, if the field had the storeTermVector flag set. If
    # termvectors had been stored with positions or offsets, a
    # TermDocPosEnumVector is returned.
    # 
    # doc_number:: document for which the term vector is returned
    # field:: field for which the term vector is returned.
    # returns:: term vector May be nil if field does not exist in the specified
    #           document or term vector was not stored.
    # raises:: IOError if index cannot be accessed
    # See Field.TermVector
    def get_term_vector(doc_number, field)
      raise NotImplementedError
    end
           
   
    # Returns +true+ if an index exists at the specified directory.  If the
    # directory does not exist or if there is no index in it.
    #
    # directory:: the directory to check for an index
    # returns:: +true+ if an index exists; +false+ otherwise
    # raises:: IOError if there is a problem with accessing the index
    def IndexReader.index_exists?(directory)
      return directory.exists?("segments")
    end

    # Returns the number of documents in this index. 
    def num_docs()
      raise NotImplementedError
    end

    # Returns one greater than the largest possible document number.
    #
    # This may be used to, e.g., determine how big to allocate an array which
    # will have an element for every document number in an index.
    def max_doc()
      raise NotImplementedError
    end

    # Returns the stored fields of the +n+<sup>th</sup>
    # +Document+ in this index. 
    def get_document(n)
      raise NotImplementedError
    end

    # Returns the first document with the term +term+. This is useful, for
    # example, if we are indexing rows from a database. We can store the id of
    # each row in a field in the index and use this method get the document by
    # the id. Hence, only one document is returned.
    #
    # term: The term we are searching for.
    def get_document_with_term(term)
      docs = term_docs_for(term)
      if (docs == nil) then return nil end
      document = nil
      begin 
        document = get_document(docs.doc) if docs.next?
      ensure 
        docs.close()
      end
      return document
    end

    # Returns true if document _n_ has been deleted 
    def deleted?(n)
      raise NotImplementedError
    end

    # Returns true if any documents have been deleted 
    def has_deletions?()
      raise NotImplementedError
    end
    
    # Returns the byte-encoded normalization factor for the named field of
    # every document.  This is used by the search code to score documents.
    # 
    # See Field#boost
    def get_norms(field, bytes=nil, offset=nil)
      raise NotImplementedError
    end

    # Expert: Resets the normalization factor for the named field of the named
    # document.  The norm represents the product of the field's Field#boost and
    # its Similarity#length_norm length normalization.  Thus, to preserve the
    # length normalization values when resetting this, one should base the new
    # value upon the old.
    # 
    # See #get_norms
    # See Similarity#decode_norm
    def set_norm(doc, field, value)
      synchronize do
        value = Similarity.encode_norm(value) if value.is_a? Float
        if(@directory_owner)
          acquire_write_lock()
        end
        do_set_norm(doc, field, value)
        @has_changes = true
      end
    end
            
    # Implements set_norm in subclass.
    def do_set_norm(doc, field, value) 
      raise NotImplementedError
    end
           
    # Returns an enumeration of all the terms in the index.
    # Each term is greater than all that precede it in the enumeration.
    def terms()
      raise NotImplementedError
    end

    # Returns an enumeration of all terms after a given term.
    #
    # Each term is greater than all that precede it in the enumeration.
    def terms_from(t)
      raise NotImplementedError
    end

    # Returns the number of documents containing the term +t+. 
    def doc_freq(t)
      raise NotImplementedError
    end

    # Returns an enumeration of all the documents which contain +term+. For each
    # document, the document number, the frequency of the term in that document
    # is also provided, for use in search scoring.  Thus, this method implements
    # the mapping:
    # 
    #   Term => <doc_num, freq><sup>*</sup>
    # 
    # The enumeration is ordered by document number.  Each document number is
    # greater than all that precede it in the enumeration.
    def term_docs_for(term)
      term_docs = term_docs()
      term_docs.seek(term)
      return term_docs
    end

    # Returns an unpositioned TermDocEnum enumerator. 
    def term_docs()
      raise NotImplementedError
    end

    # Returns an enumeration of all the documents which contain
    # +term+.  For each document, in addition to the document number
    # and frequency of the term in that document, a list of all of the ordinal
    # positions of the term in the document is available.  Thus, this method
    # implements the mapping:
    #
    #   Term => <doc_num, freq, < pos<sub>1</sub>, pos<sub>2</sub>, ...
    #   pos<sub>freq-1</sub> > > <sup>*</sup>
    #
    # This positional information faciliates phrase and proximity searching.
    # The enumeration is ordered by document number.  Each document number is
    # greater than all that precede it in the enumeration.
    def term_positions_for(term)
      term_positions = term_positions()
      term_positions.seek(term)
      return term_positions
    end

    # Returns an unpositioned @link TermDocPosEnumendenumerator. 
    def term_positions()
      raise NotImplementedError
    end

    # Tries to acquire the WriteLock on this directory.
    #
    # This method is only valid if this IndexReader is directory owner.
    # 
    # raises:: IOError If WriteLock cannot be acquired.
    def acquire_write_lock()
      if @stale
        raise IOError, "IndexReader out of date and no longer valid for delete, undelete, or set_norm operations"
      end

      if (@write_lock == nil) 
        @write_lock = @directory.make_lock(IndexWriter::WRITE_LOCK_NAME)
        if not @write_lock.obtain(IndexWriter::WRITE_LOCK_TIMEOUT) # obtain write lock
          raise IOError, "Index locked for write: " + @write_lock
        end

        # we have to check whether index has changed since this reader was opened.
        # if so, this reader is no longer valid for deletion
        if (SegmentInfos.read_current_version(@directory) > @segment_infos.version()) 
          @stale = true
          @write_lock.release()
          @write_lock = nil
          raise IOError, "IndexReader out of date and no longer valid for delete, undelete, or set_norm operations"
        end
      end
    end

    # Returns true if the reader is reading from the latest version of the
    # index.
    def latest?()
      SegmentInfos.read_current_version(@directory) == @segment_infos.version()
    end
    
    # Deletes the document numbered +doc_num+.  Once a document is deleted it
    # will not appear in TermDocEnum or TermPostitions enumerations.  Attempts to
    # read its field with the @link #documentend method will result in an error.
    # The presence of this document may still be reflected in the @link
    # #docFreqendstatistic, though this will be corrected eventually as the
    # index is further modified.
    def delete(doc_num)
      synchronize do
        acquire_write_lock() if @directory_owner
        do_delete(doc_num)
        @has_changes = true
      end
      return 1
    end

    # Implements deletion of the document numbered +doc_num+.
    # Applications should call @link #delete(int)endor @link #delete(Term)end.
    def do_delete(doc_num)
      raise NotImplementedError
    end

    # Deletes all documents containing +term+.
    # This is useful if one uses a document field to hold a unique ID string for
    # the document.  Then to delete such a document, one merely constructs a
    # term with the appropriate field and the unique ID string as its text and
    # passes it to this method.  Returns the number of documents deleted.  See
    # #delete for information about when this deletion will become effective.
    def delete_docs_with_term(term)
      docs = term_docs_for(term)
      if (docs == nil) then return 0 end
      n = 0
      begin 
        while (docs.next?) 
          delete(docs.doc)
          n += 1
        end
      ensure 
        docs.close()
      end
      return n
    end

    # Undeletes all documents currently marked as deleted in this index.
    def undelete_all()
      synchronize do
        acquire_write_lock() if @directory_owner
        do_undelete_all()
        @has_changes = true
      end
    end
    
    # Commit changes resulting from delete, undelete_all, or set_norm operations
    # 
    # raises:: IOError
    def commit()
      synchronize do
        if @has_changes
          if @directory_owner
            @directory.synchronize do # in- & inter-process sync
              commit_lock = @directory.make_lock(IndexWriter::COMMIT_LOCK_NAME)
              commit_lock.while_locked do
                do_commit()
                @segment_infos.write(@directory)
              end
            end
            if (@write_lock != nil) 
              @write_lock.release()  # release write lock
              @write_lock = nil
            end
          else
            do_commit()
          end
        end
        @has_changes = false
      end
    end
    
    # Closes files associated with this index.
    # Also saves any new deletions to disk.
    # No other methods should be called after this has been called.
    def close()
      synchronize do
        commit()
        do_close()
        @directory.close() if @close_directory
      end
    end

    protected 

    # Implements actual undelete_all() in subclass. 
    def do_undelete_all()
      raise NotImplementedError
    end

    # Implements commit. 
    def do_commit()
      raise NotImplementedError
    end
    

    # Implements close. 
    def do_close()
      raise NotImplementedError
    end

    # Get a list of unique field names that exist in this index and have the
    # specified field option information.
    # fld_option:: specifies which field option should be available for the
    #              returned fields
    # returns:: Collection of Strings indicating the names of the fields.
    # See IndexReader.FieldOption
    def get_field_names()
      raise NotImplementedError
    end

    # Returns +true+ iff the index in the named directory is
    # currently locked.
    # directory:: the directory to check for a lock
    # raises:: IOError if there is a problem with accessing the index
    def IndexReader.locked?(directory)
      return (directory.make_lock(IndexWriter::WRITE_LOCK_NAME).locked? or
        directory.make_lock(IndexWriter::COMMIT_LOCK_NAME).locked?)
    end

    # Forcibly unlocks the index in the named directory.
    #
    # Caution: this should only be used by failure recovery code,
    # when it is known that no other process nor thread is in fact
    # currently accessing this index.
    def IndexReader.unlock(directory)
      directory.make_lock(IndexWriter::WRITE_LOCK_NAME).release
      directory.make_lock(IndexWriter::COMMIT_LOCK_NAME).release
    end
  end
end
<commit_msg>fixed bug in IndexReader to stop it wiping out the index when opened with a string argument.<commit_after>require 'monitor'

module Ferret::Index
  # IndexReader is an abstract class, providing an interface for accessing an
  # index.  Search of an index is done entirely through this abstract interface,
  # class which implements it is searchable.
  #
  # Concrete subclasses of IndexReader are usually constructed with a call to
  # one of the static <tt>open()</tt> methods, e.g. <tt>#open</tt>.
  #
  # For efficiency, in this API documents are often referred to via
  # _document numbers_, non-negative integers which each name a unique
  # document in the index.  These document numbers are ephemeral, ie they may change
  # as documents are added to and deleted from an index.  Clients should thus not
  # rely on a given document having the same number between sessions.
  # 
  # An IndexReader can be opened on a directory for which an IndexWriter is
  # opened already, but it cannot be used to delete documents from the index then.
  class IndexReader 
    include MonitorMixin
    
    # This array contains all filename extensions used by Lucene's index files, with
    # one exception, namely the extension made up from +.f+ + a number.
    # Also note that two of Lucene's files (+deletable+ and
    # +segments+) don't have any filename extension.
    FILENAME_EXTENSIONS = ["cfs",
                           "fnm",
                           "fdx",
                           "fdt",
                           "tii",
                           "tis",
                           "frq",
                           "prx",
                           "del",
                           "tvx",
                           "tvd",
                           "tvf",
                           "tvp"]

    attr_reader :directory
    
    class FieldOption < Ferret::Utils::Parameter
      # all fields
      ALL = FieldOption.new("ALL")
      # all indexed fields
      INDEXED = FieldOption.new("INDEXED")
      # all fields which are not indexed
      UNINDEXED = FieldOption.new("UNINDEXED")
      # all fields which are indexed with termvectors enables
      INDEXED_WITH_TERM_VECTOR = FieldOption.new("INDEXED_WITH_TERM_VECTOR")
      # all fields which are indexed but don't have termvectors enabled
      INDEXED_NO_TERM_VECTOR = FieldOption.new("INDEXED_NO_TERM_VECTOR")
      # all fields where termvectors are enabled. Please note that only standard
      # termvector fields are returned
      TERM_VECTOR = FieldOption.new("TERM_VECTOR")
      # all field with termvectors wiht positions enabled
      TERM_VECTOR_WITH_POSITION = FieldOption.new("TERM_VECTOR_WITH_POSITION")
      # all fields where termvectors with offset position are set
      TERM_VECTOR_WITH_OFFSET = FieldOption.new("TERM_VECTOR_WITH_OFFSET")
      # all fields where termvectors with offset and position values set
      TERM_VECTOR_WITH_POSITION_OFFSET =
        FieldOption.new("TERM_VECTOR_WITH_POSITION_OFFSET")
    end
    
    # To create an IndexReader use the IndexReader.open method. This method
    # should only be used by subclasses.
    #
    # directory::       Directory where IndexReader files reside.
    # segment_infos::   Used for write-l
    # close_directory:: close the directory when the index reader is closed
    def initialize(directory, segment_infos = nil,
                   close_directory = false, directory_owner = false)
      super()
      @directory = directory
      @close_directory = close_directory
      @segment_infos = segment_infos
      @directory_owner = directory_owner

      @has_changes = false
      @stale = false
      @write_lock = nil

      #ObjectSpace.define_finalizer(self, lambda { |id| @write_lock.release() if @write_lock})
    end

    # Returns an index reader to read the index in the directory
    #
    # directory::       This can either be a Directory object or you can pass
    #                   nil (RamDirectory is created) or a path (FSDirectory
    #                   is created). If you chose the second or third options,
    #                   you should leave close_directory as true and infos as
    #                   nil.
    # close_directory:: True if you want the IndexReader to close the
    #                   directory when the IndexReader is closed. You'll want
    #                   to set this to false if other objects are using the
    #                   same directory object.
    # infos::           Expert: This can be used to read an different version
    #                   of the index but should really be left alone.
    def IndexReader.open(directory, close_directory = true, infos = nil)
      if directory.nil?
        directory = Ferret::Store::RAMDirectory.new
      elsif directory.is_a?(String)
        directory = Ferret::Store::FSDirectory.new(directory, false)
      end
      directory.synchronize do # in- & inter-process sync
        commit_lock = directory.make_lock(IndexWriter::COMMIT_LOCK_NAME)
        commit_lock.while_locked() do
          if infos.nil?
            infos = SegmentInfos.new()
            infos.read(directory)
          end
          if (infos.size() == 1) # index is optimized
            return SegmentReader.get(infos[0], infos, close_directory)
          end
          readers = Array.new(infos.size)
          infos.size.times do |i|
            readers[i] = SegmentReader.get(infos[i])
          end
          return MultiReader.new(readers, directory, infos, close_directory)
        end
      end
    end

    # Reads version number from segments files. The version number counts the
    # number of changes of the index.
    # 
    # directory:: where the index resides.
    # returns:: version number.
    # raises:: IOError if segments file cannot be read.
    def IndexReader.get_current_version(directory)
      return SegmentInfos.read_current_version(directory)
    end

    # Return an array of term vectors for the specified document.  The array
    # contains a vector for each vectorized field in the document.  Each vector
    # contains terms and frequencies for all terms in a given vectorized field.
    # If no such fields existed, the method returns nil. The term vectors that
    # are returned my either be of type TermFreqVector or of type
    # TermDocPosEnumVector if positions or offsets have been stored.
    # 
    # doc_number:: document for which term vectors are returned
    # returns:: array of term vectors. May be nil if no term vectors have been
    #           stored for the specified document.
    # raises:: IOError if index cannot be accessed
    #
    # See Field.TermVector
    def get_term_vectors(doc_number)
      raise NotImplementedError
    end
           

    
    # Return a term vector for the specified document and field. The returned
    # vector contains terms and frequencies for the terms in the specified
    # field of this document, if the field had the storeTermVector flag set. If
    # termvectors had been stored with positions or offsets, a
    # TermDocPosEnumVector is returned.
    # 
    # doc_number:: document for which the term vector is returned
    # field:: field for which the term vector is returned.
    # returns:: term vector May be nil if field does not exist in the specified
    #           document or term vector was not stored.
    # raises:: IOError if index cannot be accessed
    # See Field.TermVector
    def get_term_vector(doc_number, field)
      raise NotImplementedError
    end
           
   
    # Returns +true+ if an index exists at the specified directory.  If the
    # directory does not exist or if there is no index in it.
    #
    # directory:: the directory to check for an index
    # returns:: +true+ if an index exists; +false+ otherwise
    # raises:: IOError if there is a problem with accessing the index
    def IndexReader.index_exists?(directory)
      return directory.exists?("segments")
    end

    # Returns the number of documents in this index. 
    def num_docs()
      raise NotImplementedError
    end

    # Returns one greater than the largest possible document number.
    #
    # This may be used to, e.g., determine how big to allocate an array which
    # will have an element for every document number in an index.
    def max_doc()
      raise NotImplementedError
    end

    # Returns the stored fields of the +n+<sup>th</sup>
    # +Document+ in this index. 
    def get_document(n)
      raise NotImplementedError
    end

    # Returns the first document with the term +term+. This is useful, for
    # example, if we are indexing rows from a database. We can store the id of
    # each row in a field in the index and use this method get the document by
    # the id. Hence, only one document is returned.
    #
    # term: The term we are searching for.
    def get_document_with_term(term)
      docs = term_docs_for(term)
      if (docs == nil) then return nil end
      document = nil
      begin 
        document = get_document(docs.doc) if docs.next?
      ensure 
        docs.close()
      end
      return document
    end

    # Returns true if document _n_ has been deleted 
    def deleted?(n)
      raise NotImplementedError
    end

    # Returns true if any documents have been deleted 
    def has_deletions?()
      raise NotImplementedError
    end
    
    # Returns the byte-encoded normalization factor for the named field of
    # every document.  This is used by the search code to score documents.
    # 
    # See Field#boost
    def get_norms(field, bytes=nil, offset=nil)
      raise NotImplementedError
    end

    # Expert: Resets the normalization factor for the named field of the named
    # document.  The norm represents the product of the field's Field#boost and
    # its Similarity#length_norm length normalization.  Thus, to preserve the
    # length normalization values when resetting this, one should base the new
    # value upon the old.
    # 
    # See #get_norms
    # See Similarity#decode_norm
    def set_norm(doc, field, value)
      synchronize do
        value = Similarity.encode_norm(value) if value.is_a? Float
        if(@directory_owner)
          acquire_write_lock()
        end
        do_set_norm(doc, field, value)
        @has_changes = true
      end
    end
            
    # Implements set_norm in subclass.
    def do_set_norm(doc, field, value) 
      raise NotImplementedError
    end
           
    # Returns an enumeration of all the terms in the index.
    # Each term is greater than all that precede it in the enumeration.
    def terms()
      raise NotImplementedError
    end

    # Returns an enumeration of all terms after a given term.
    #
    # Each term is greater than all that precede it in the enumeration.
    def terms_from(t)
      raise NotImplementedError
    end

    # Returns the number of documents containing the term +t+. 
    def doc_freq(t)
      raise NotImplementedError
    end

    # Returns an enumeration of all the documents which contain +term+. For each
    # document, the document number, the frequency of the term in that document
    # is also provided, for use in search scoring.  Thus, this method implements
    # the mapping:
    # 
    #   Term => <doc_num, freq><sup>*</sup>
    # 
    # The enumeration is ordered by document number.  Each document number is
    # greater than all that precede it in the enumeration.
    def term_docs_for(term)
      term_docs = term_docs()
      term_docs.seek(term)
      return term_docs
    end

    # Returns an unpositioned TermDocEnum enumerator. 
    def term_docs()
      raise NotImplementedError
    end

    # Returns an enumeration of all the documents which contain
    # +term+.  For each document, in addition to the document number
    # and frequency of the term in that document, a list of all of the ordinal
    # positions of the term in the document is available.  Thus, this method
    # implements the mapping:
    #
    #   Term => <doc_num, freq, < pos<sub>1</sub>, pos<sub>2</sub>, ...
    #   pos<sub>freq-1</sub> > > <sup>*</sup>
    #
    # This positional information faciliates phrase and proximity searching.
    # The enumeration is ordered by document number.  Each document number is
    # greater than all that precede it in the enumeration.
    def term_positions_for(term)
      term_positions = term_positions()
      term_positions.seek(term)
      return term_positions
    end

    # Returns an unpositioned @link TermDocPosEnumendenumerator. 
    def term_positions()
      raise NotImplementedError
    end

    # Tries to acquire the WriteLock on this directory.
    #
    # This method is only valid if this IndexReader is directory owner.
    # 
    # raises:: IOError If WriteLock cannot be acquired.
    def acquire_write_lock()
      if @stale
        raise IOError, "IndexReader out of date and no longer valid for delete, undelete, or set_norm operations"
      end

      if (@write_lock == nil) 
        @write_lock = @directory.make_lock(IndexWriter::WRITE_LOCK_NAME)
        if not @write_lock.obtain(IndexWriter::WRITE_LOCK_TIMEOUT) # obtain write lock
          raise IOError, "Index locked for write: " + @write_lock
        end

        # we have to check whether index has changed since this reader was opened.
        # if so, this reader is no longer valid for deletion
        if (SegmentInfos.read_current_version(@directory) > @segment_infos.version()) 
          @stale = true
          @write_lock.release()
          @write_lock = nil
          raise IOError, "IndexReader out of date and no longer valid for delete, undelete, or set_norm operations"
        end
      end
    end

    # Returns true if the reader is reading from the latest version of the
    # index.
    def latest?()
      SegmentInfos.read_current_version(@directory) == @segment_infos.version()
    end
    
    # Deletes the document numbered +doc_num+.  Once a document is deleted it
    # will not appear in TermDocEnum or TermPostitions enumerations.  Attempts to
    # read its field with the @link #documentend method will result in an error.
    # The presence of this document may still be reflected in the @link
    # #docFreqendstatistic, though this will be corrected eventually as the
    # index is further modified.
    def delete(doc_num)
      synchronize do
        acquire_write_lock() if @directory_owner
        do_delete(doc_num)
        @has_changes = true
      end
      return 1
    end

    # Implements deletion of the document numbered +doc_num+.
    # Applications should call @link #delete(int)endor @link #delete(Term)end.
    def do_delete(doc_num)
      raise NotImplementedError
    end

    # Deletes all documents containing +term+.
    # This is useful if one uses a document field to hold a unique ID string for
    # the document.  Then to delete such a document, one merely constructs a
    # term with the appropriate field and the unique ID string as its text and
    # passes it to this method.  Returns the number of documents deleted.  See
    # #delete for information about when this deletion will become effective.
    def delete_docs_with_term(term)
      docs = term_docs_for(term)
      if (docs == nil) then return 0 end
      n = 0
      begin 
        while (docs.next?) 
          delete(docs.doc)
          n += 1
        end
      ensure 
        docs.close()
      end
      return n
    end

    # Undeletes all documents currently marked as deleted in this index.
    def undelete_all()
      synchronize do
        acquire_write_lock() if @directory_owner
        do_undelete_all()
        @has_changes = true
      end
    end
    
    # Commit changes resulting from delete, undelete_all, or set_norm operations
    # 
    # raises:: IOError
    def commit()
      synchronize do
        if @has_changes
          if @directory_owner
            @directory.synchronize do # in- & inter-process sync
              commit_lock = @directory.make_lock(IndexWriter::COMMIT_LOCK_NAME)
              commit_lock.while_locked do
                do_commit()
                @segment_infos.write(@directory)
              end
            end
            if (@write_lock != nil) 
              @write_lock.release()  # release write lock
              @write_lock = nil
            end
          else
            do_commit()
          end
        end
        @has_changes = false
      end
    end
    
    # Closes files associated with this index.
    # Also saves any new deletions to disk.
    # No other methods should be called after this has been called.
    def close()
      synchronize do
        commit()
        do_close()
        @directory.close() if @close_directory
      end
    end

    protected 

    # Implements actual undelete_all() in subclass. 
    def do_undelete_all()
      raise NotImplementedError
    end

    # Implements commit. 
    def do_commit()
      raise NotImplementedError
    end
    

    # Implements close. 
    def do_close()
      raise NotImplementedError
    end

    # Get a list of unique field names that exist in this index and have the
    # specified field option information.
    # fld_option:: specifies which field option should be available for the
    #              returned fields
    # returns:: Collection of Strings indicating the names of the fields.
    # See IndexReader.FieldOption
    def get_field_names()
      raise NotImplementedError
    end

    # Returns +true+ iff the index in the named directory is
    # currently locked.
    # directory:: the directory to check for a lock
    # raises:: IOError if there is a problem with accessing the index
    def IndexReader.locked?(directory)
      return (directory.make_lock(IndexWriter::WRITE_LOCK_NAME).locked? or
        directory.make_lock(IndexWriter::COMMIT_LOCK_NAME).locked?)
    end

    # Forcibly unlocks the index in the named directory.
    #
    # Caution: this should only be used by failure recovery code,
    # when it is known that no other process nor thread is in fact
    # currently accessing this index.
    def IndexReader.unlock(directory)
      directory.make_lock(IndexWriter::WRITE_LOCK_NAME).release
      directory.make_lock(IndexWriter::COMMIT_LOCK_NAME).release
    end
  end
end

{'fraction_non_alphanumeric': 0.057169559412550065, 'fraction_numerical': 0.0004806408544726302, 'mean_word_length': 2.1878617637044604, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1342395', 'n_tokens_mistral': 10467, 'n_tokens_neox': 9602, 'n_words': 3904}