<commit_before>;;; ------------------------------------------------- -*- Mode: LISP -*-

(in-package :asdf)

(defsystem #:montezuma
    :name "Montezuma"
    :author "<NAME> <<EMAIL>>"
    :maintainer "<NAME> <<EMAIL>>"
    :version "0.1"
    :licence "MIT"
    :description "Montezuma is a port of the Lucene text search engine library."
    :long-description ""

    :depends-on ("cl-ppcre" "cl-fad")

    :components
    ((:file "package")
     (:module "util"
       :components ((:file "while")
		    (:file "porter-stemmer")
		    (:file "streams")
		    (:file "pipes")
		    (:file "mop")
		    (:file "priority-queue"      :depends-on ("while"))
		    (:file "strings")
		    (:file "tables")
		    (:file "bit-vector")
		    (:file "files"))
       :depends-on ("package"))
     (:module "store"
       :components ((:file "directory")
		    (:file "index-io")
		    (:file "buffered-index-io"   :depends-on ("index-io"))
		    (:file "ram-store"           :depends-on ("buffered-index-io" "directory"))
		    (:file "fs-store"            :depends-on ("buffered-index-io" "directory")))
       :depends-on ("package" "util"))
     (:module "document"
       :components ((:file "field")
		    (:file "document"))
       :depends-on ("package"))
     (:module "analysis"
       :components ((:file "token")
		    (:file "token-stream")
		    (:file "token-filters"       :depends-on ("token" "token-stream"))
		    (:file "tokenizers"          :depends-on ("token" "token-stream"))
		    (:file "standard-tokenizer"  :depends-on ("tokenizers"))
		    (:file "analyzers"           :depends-on ("standard-tokenizer")))
       :depends-on ("package" "util"))
     (:module "search"
       :components ((:file "similarity")))
     (:module "index"
       :components ((:file "index-filenames")
		    (:file "term")
		    (:file "term-info")
		    (:file "term-buffer"         :depends-on ("term"))
		    (:file "field-infos")
		    (:file "term-enum")
		    (:file "term-doc-enum")
		    (:file "term-infos-io")
		    (:file "multiple-term-doc-pos-enum")
		    (:file "term-vector-offset-info")
		    (:file "segment-term-vector")
		    (:file "term-vectors-io"     :depends-on ("segment-term-vector"))
		    (:file "segment-term-enum"   :depends-on ("term-infos-io"))
		    (:file "fields-io")
		    (:file "compound-file-io")
		    (:file "segment-merge-info")
		    (:file "segment-merge-queue")
		    (:file "segment-infos")
		    (:file "segment-reader"      :depends-on ("index-reader"))
		    (:file "multi-reader"        :depends-on ("index-reader"))
		    (:file "index-writer"        :depends-on ("segment-reader"))
		    (:file "document-writer")
		    (:file "segment-merger"      :depends-on ("index-filenames" "fields-io" "term-buffer"))
		    (:file "index-reader")
		    (:file "index"))
       :depends-on ("analysis" "search"))))

(defmethod perform ((o test-op) (c (eql (find-system '#:montezuma))))
  (oos 'load-op '#:montezuma-tests)
  (oos 'test-op '#:montezuma-tests :force t))



(defsystem #:montezuma-tests
  :depends-on (#:montezuma)
  :components
  ((:module "tests"
     :components
      ((:module "unit"
	  :components ((:file "tests")
		       (:module "util"
			 :components ((:file "tc-priority-queue")
				      (:file "tc-tables"))
			 :depends-on ("tests"))
		       (:module "store"
			 :components ((:file "tc-store")
				      (:file "tc-ram-store")
				      (:file "tc-fs-store"))
			 :depends-on ("tests"))
		       (:module "document"
			 :components ((:file "tc-field")
				      (:file "tc-document"))
			 :depends-on ("tests"))
		       (:module "analysis"
			 :components ((:file "tc-lowercase-filter")
				      (:file "tc-stop-filter")
				      (:file "tc-porter-stem-filter")
				      (:file "tc-letter-tokenizer")
				      (:file "tc-whitespace-tokenizer")
				      (:file "tc-lowercase-tokenizer")
				      (:file "tc-standard-tokenizer")
				      (:file "tc-analyzer")
				      (:file "tc-stop-analyzer")
				      (:file "tc-whitespace-analyzer")
				      (:file "tc-standard-analyzer"))
			 :depends-on ("tests"))
		       (:module "index"
			 :components ((:file "tc-term")
				      (:file "tc-term-info")
				      (:file "tc-term-buffer")
				      (:file "tc-field-infos")
				      (:file "tc-term-infos-io")
				      (:file "tc-term-vectors-io")
				      (:file "tc-fields-io")
				      (:file "tc-compound-file-io")
				      (:file "tc-segment-term-enum")
				      (:file "tc-segment-term-vector")
				      (:file "tc-segment-infos")
				      (:file "th-doc")
				      (:file "tc-index-writer" :depends-on ("th-doc"))
				      (:file "tc-index-reader" :depends-on ("th-doc"))
				      (:file "tc-multiple-term-doc-pos-enum" :depends-on ("th-doc")))<commit_msg>Added search to system and initial search tests to test system<commit_after>;;; ------------------------------------------------- -*- Mode: LISP -*-

(in-package :asdf)

(defsystem #:montezuma
    :name "Montezuma"
    :author "<NAME> <<EMAIL>>"
    :maintainer "<NAME> <<EMAIL>>"
    :version "0.1"
    :licence "MIT"
    :description "Montezuma is a port of the Lucene text search engine library."
    :long-description ""

    :depends-on ("cl-ppcre" "cl-fad")

    :components
    ((:file "package")
     (:module "util"
       :components ((:file "while")
		    (:file "porter-stemmer")
		    (:file "streams")
		    (:file "pipes")
		    (:file "mop")
		    (:file "priority-queue"      :depends-on ("while"))
		    (:file "strings")
		    (:file "tables")
		    (:file "bit-vector")
		    (:file "files")
                    (:file "comparable"))
       :depends-on ("package"))
     (:module "store"
       :components ((:file "directory")
		    (:file "index-io")
		    (:file "buffered-index-io"   :depends-on ("index-io"))
		    (:file "ram-store"           :depends-on ("buffered-index-io" "directory"))
		    (:file "fs-store"            :depends-on ("buffered-index-io" "directory")))
       :depends-on ("package" "util"))
     (:module "document"
       :components ((:file "field")
		    (:file "document"))
       :depends-on ("package"))
     (:module "analysis"
       :components ((:file "token")
		    (:file "token-stream")
		    (:file "token-filters"       :depends-on ("token" "token-stream"))
		    (:file "tokenizers"          :depends-on ("token" "token-stream"))
		    (:file "standard-tokenizer"  :depends-on ("tokenizers"))
		    (:file "analyzers"           :depends-on ("standard-tokenizer")))
       :depends-on ("package" "util"))
     (:module "search"
              :components ((:file "api")
                           (:file "similarity")
                           (:file "boolean-clause")
                           (:file "scorer")
                           (:file "score-doc")
                           (:file "score-doc-comparator")
                           (:file "weight")
                           (:file "query")
                           (:file "term-query"))
              :depends-on ("package"))
     (:module "index"
       :components ((:file "index-filenames")
		    (:file "term")
		    (:file "term-info")
		    (:file "term-buffer"         :depends-on ("term"))
		    (:file "field-infos")
		    (:file "term-enum")
		    (:file "term-doc-enum")
		    (:file "term-infos-io")
		    (:file "multiple-term-doc-pos-enum")
		    (:file "term-vector-offset-info")
		    (:file "segment-term-vector")
		    (:file "term-vectors-io"     :depends-on ("segment-term-vector"))
		    (:file "segment-term-enum"   :depends-on ("term-infos-io"))
		    (:file "fields-io")
		    (:file "compound-file-io")
		    (:file "segment-merge-info")
		    (:file "segment-merge-queue")
		    (:file "segment-infos")
		    (:file "segment-reader"      :depends-on ("index-reader"))
		    (:file "multi-reader"        :depends-on ("index-reader"))
		    (:file "index-writer"        :depends-on ("segment-reader"))
		    (:file "document-writer")
		    (:file "segment-merger"      :depends-on ("index-filenames" "fields-io" "term-buffer"))
		    (:file "index-reader")
		    (:file "index"))
       :depends-on ("analysis" "search"))))

(defmethod perform ((o test-op) (c (eql (find-system '#:montezuma))))
  (oos 'load-op '#:montezuma-tests)
  (oos 'test-op '#:montezuma-tests :force t))



(defsystem #:montezuma-tests
  :depends-on (#:montezuma)
  :components
  ((:module "tests"
     :components
      ((:module "unit"
	  :components ((:file "tests")
		       (:module "util"
			 :components ((:file "tc-priority-queue")
				      (:file "tc-tables"))
			 :depends-on ("tests"))
		       (:module "store"
			 :components ((:file "tc-store")
				      (:file "tc-ram-store")
				      (:file "tc-fs-store"))
			 :depends-on ("tests"))
		       (:module "document"
			 :components ((:file "tc-field")
				      (:file "tc-document"))
			 :depends-on ("tests"))
		       (:module "analysis"
			 :components ((:file "tc-lowercase-filter")
				      (:file "tc-stop-filter")
				      (:file "tc-porter-stem-filter")
				      (:file "tc-letter-tokenizer")
				      (:file "tc-whitespace-tokenizer")
				      (:file "tc-lowercase-tokenizer")
				      (:file "tc-standard-tokenizer")
				      (:file "tc-analyzer")
				      (:file "tc-stop-analyzer")
				      (:file "tc-whitespace-analyzer")
				      (:file "tc-standard-analyzer"))
			 :depends-on ("tests"))
                       (:module "search"
                         :components ((:file "tc-similarity"))
                         :depends-on ("tests"))
		       (:module "index"
			 :components ((:file "tc-term")
				      (:file "tc-term-info")
				      (:file "tc-term-buffer")
				      (:file "tc-field-infos")
				      (:file "tc-term-infos-io")
				      (:file "tc-term-vectors-io")
				      (:file "tc-fields-io")
				      (:file "tc-compound-file-io")
				      (:file "tc-segment-term-enum")
				      (:file "tc-segment-term-vector")
				      (:file "tc-segment-infos")
				      (:file "th-doc")
				      (:file "tc-index-writer" :depends-on ("th-doc"))
				      (:file "tc-index-reader" :depends-on ("th-doc"))
				      (:file "tc-multiple-term-doc-pos-enum" :depends-on ("th-doc")))
{'fraction_non_alphanumeric': 0.2043709391612522, 'fraction_numerical': 0.00039377830281551486, 'mean_word_length': 2.0991458206223306, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7522178', 'n_tokens_mistral': 3902, 'n_tokens_neox': 3223, 'n_words': 690}