<commit_before>(in-package montezuma)

(defclass index-searcher ()
  ((similarity :accessor similarity :initarg :similarity)
   (reader :accessor reader)
   (directory :accessor directory))
  (:default-initargs
    :similarity (make-default-similarity)))

(defmethod initialize-instance :after ((self index-searcher) &key reader)
  (setf (slot-value self 'reader) (initialize-reader self reader)))

(defmethod initialize-reader ((self index-searcher) (reader index-reader))
  (values reader))

(defmethod initialize-reader ((self index-searcher) (reader directory))
  (open-index-reader reader :close-directory-p false))

(defmethod initialize-reader ((self index-searcher) (reader string))
  (setf (directory self) (make-fs-directory reader))
  (open-index-reader (directory self) :close-directory-p t))

(defmethod close ((self index-searcher))
  ;; delegate
  (close (reader self)))

(defmethod term-doc-freq ((self index-searcher) (term term))
  ;; delegate
  (term-doc-freq (reader self) term))

(defmethod term-doc-freqs ((self index-searcher) (terms sequence))
  (let ((result (make-array (length terms))))
    (dosequence (i terms)
      (setf (aref i result) 
            (term-doc-freq self (aref terms i))))
    (values result)))

(defmethod get-document ((self index-searcher) index)
  ;; delegate
  (get-document (reader self) index))

(defmethod max-doc ((self index-searcher))
  ;; delegate
  (max-doc (reader self)))

(defmethod create-weight ((self index-searcher) (query query))
  (weight query self))

(defmethod search ((self index-searcher) (query query) 
                   &optional options)
  (destructuring-bind (&key (filter nil) (first-document 0) (num-documents 10)
                            (max-size (+ first-document num-documents))
                            (sort nil)) options
    
    (assert (plusp num-documents))
    (assert (not (minusp first-document)))
    (let ((scorer (scorer (weight query self) (reader self))))
      (when (null scorer)
        (return-from search (make-instance 'top-docs)))
      )))

(defmethod search-each ((self index-searcher) (query query) &optional (options nil))
  )

(defmethod rewrite ((self index-searcher) original)
  (let* ((query original)
         (rewritten-query (rewrite query (reader self))))
    (while (not (equal query rewritten-query))
      (setf query rewritten-query
            rewritten-query (rewrite query (reader self))))
    (values query)))

(defmethod explain ((self index-searcher) (query query) index)
  ;; not implemented
  (error "not yet implemented"))<commit_msg>Finished search, search-each still WIP<commit_after>(in-package montezuma)

(defclass index-searcher ()
  ((similarity :accessor similarity :initarg :similarity)
   (reader :accessor reader)
   (directory :accessor directory))
  (:default-initargs
    :similarity (make-default-similarity)))

(defmethod initialize-instance :after ((self index-searcher) &key reader)
  (setf (slot-value self 'reader) (initialize-reader self reader)))

(defmethod initialize-reader ((self index-searcher) (reader index-reader))
  (values reader))

(defmethod initialize-reader ((self index-searcher) (reader directory))
  (open-index-reader reader :close-directory-p false))

(defmethod initialize-reader ((self index-searcher) (reader string))
  (setf (directory self) (make-fs-directory reader))
  (open-index-reader (directory self) :close-directory-p t))

(defmethod close ((self index-searcher))
  ;; delegate
  (close (reader self)))

(defmethod term-doc-freq ((self index-searcher) (term term))
  ;; delegate
  (term-doc-freq (reader self) term))

(defmethod term-doc-freqs ((self index-searcher) (terms sequence))
  (let ((result (make-array (length terms))))
    (dosequence (i terms)
      (setf (aref i result) 
            (term-doc-freq self (aref terms i))))
    (values result)))

(defmethod get-document ((self index-searcher) index)
  ;; delegate
  (get-document (reader self) index))

(defmethod max-doc ((self index-searcher))
  ;; delegate
  (max-doc (reader self)))

(defmethod create-weight ((self index-searcher) (query query))
  (weight query self))

(defmethod search ((self index-searcher) (query query) 
                   &optional options)
  (destructuring-bind (&key (filter nil) (first-document 0) (num-documents 10)
                            (max-size (+ first-document num-documents))
                            (sort nil)) options
    
    (assert (plusp num-documents))
    (assert (not (minusp first-document)))
    (let ((scorer (scorer (weight query self) (reader self))))
      (when (null scorer)
        (return-from search (make-instance 'top-docs)))
      
      ;;?? ignore filter
      ;;?? ignore sort
      (let ((hq (make-instance 'hit-queue))
            (total-hits 0)
            (minimum-score 0.0))
        (each-hit scorer 
                  (lambda (doc score)
                    (when (and (plusp score)
                               ;; bits
                               )
                      (incf total-hits)
                      (when (or (< (size hq) max-size)
                                (>= score minimum-score))
                        (queue-push hq (make-instance 'score-doc :doc doc :score score))
                        (setf minimum-score (score (queue-top hq)))))))
        
        (let ((score-docs (make-array 10 :fill-pointer 0 :adjustable t)))
          (when (> (size hq) first-document)
            (when (< (- (size hq) first-document) num-documents)
              (setf num-documents (- (size hq) first-document)))
            (dotimes (i num-documents)
              (vector-push-extend (queue-pop hq) score-docs)))
          (queue-clear hq)
        
          (values (make-instance 'top-docs :total-hits total-hits
                                 :score-docs score-docs)))))))

(defmethod search-each ((self index-searcher) (query query) &optional (options nil))
  (let ((scorer (scorer (weight query self) (reader self))))
    (when (null scorer)
      (return-from search-each nil))
    
    ;;?? bits
    (each-hit scorer
              (lambda (doc score)
                (if (and (plusp score)
                         ;;?? bits
                         )
                  (yield doc score))))))
    
(defmethod rewrite ((self index-searcher) original)
  (let* ((query original)
         (rewritten-query (rewrite query (reader self))))
    (while (not (equal query rewritten-query))
      (setf query rewritten-query
            rewritten-query (rewrite query (reader self))))
    (values query)))

(defmethod explain ((self index-searcher) (query query) index)
  ;; not implemented
  (error "not yet implemented"))
{'fraction_non_alphanumeric': 0.13594124700239807, 'fraction_numerical': 0.0017985611510791368, 'mean_word_length': 2.694905869324474, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2347815', 'n_tokens_mistral': 2002, 'n_tokens_neox': 1947, 'n_words': 575}