<commit_before>;;; ------------------------------------------------- -*- Mode: LISP -*-

(in-package :asdf)

(defsystem #:montezuma
    :name "Montezuma"
    :author "<NAME> <<EMAIL>>"
    :maintainer "<NAME> <<EMAIL>>"
    :version "0.1"
    :licence "MIT"
    :description ""
    :long-description ""

    :depends-on ("cl-ppcre")

    :components
    ((:file "package")
     (:module "util"
	      :components ((:file "porter-stemmer")
			   (:file "streams"))
	      :depends-on ("package"))
     (:module "document"
	      :components ((:file "field")
			   (:file "document"))
	      :depends-on ("package"))
     (:module "analysis"
	      :components ((:file "token")
			   (:file "token-stream")
			   (:file "token-filters" :depends-on ("token" "token-stream"))
			   (:file "tokenizers"    :depends-on ("token" "token-stream")))
	      :depends-on ("package" "util"))))

(defmethod perform ((o test-op) (c (eql (find-system '#:montezuma))))
  (oos 'load-op '#:montezuma-tests)
  (oos 'test-op '#:montezuma-tests :force t))



(defsystem #:montezuma-tests
  :depends-on (#:montezuma)
  :components
  ((:module "tests"
	    :components
	    ((:module "unit"
		      :components
		      ((:file "tests")
		       (:module "document"
				:components ((:file "field")
					     (:file "document"))
				:depends-on ("tests"))
		       (:module "analysis"
				:components ((:file "letter-tokenizer")
					     (:file "lowercase-filter"))
				:depends-on ("tests"))))))))

(defmethod asdf:perform ((o asdf:test-op) (c (eql (find-system '#:montezuma-tests))))
  (or (funcall (intern (symbol-name '#:run-tests)
                       (find-package '#:montezuma)))
      (error "test-op failed")))<commit_msg>Added unit tests for lowercase-tokenizer.<commit_after>;;; ------------------------------------------------- -*- Mode: LISP -*-

(in-package :asdf)

(defsystem #:montezuma
    :name "Montezuma"
    :author "<NAME> <<EMAIL>>"
    :maintainer "<NAME> <<EMAIL>>"
    :version "0.1"
    :licence "MIT"
    :description ""
    :long-description ""

    :depends-on ("cl-ppcre")

    :components
    ((:file "package")
     (:module "util"
	      :components ((:file "porter-stemmer")
			   (:file "streams"))
	      :depends-on ("package"))
     (:module "document"
	      :components ((:file "field")
			   (:file "document"))
	      :depends-on ("package"))
     (:module "analysis"
	      :components ((:file "token")
			   (:file "token-stream")
			   (:file "token-filters" :depends-on ("token" "token-stream"))
			   (:file "tokenizers"    :depends-on ("token" "token-stream")))
	      :depends-on ("package" "util"))))

(defmethod perform ((o test-op) (c (eql (find-system '#:montezuma))))
  (oos 'load-op '#:montezuma-tests)
  (oos 'test-op '#:montezuma-tests :force t))



(defsystem #:montezuma-tests
  :depends-on (#:montezuma)
  :components
  ((:module "tests"
	    :components
	    ((:module "unit"
		      :components
		      ((:file "tests")
		       (:module "document"
				:components ((:file "field")
					     (:file "document"))
				:depends-on ("tests"))
		       (:module "analysis"
				:components ((:file "letter-tokenizer")
					     (:file "lowercase-filter")
					     (:file "lowercase-tokenizer"))
				:depends-on ("tests"))))))))

(defmethod asdf:perform ((o asdf:test-op) (c (eql (find-system '#:montezuma-tests))))
  (or (funcall (intern (symbol-name '#:run-tests)
                       (find-package '#:montezuma)))
      (error "test-op failed")))
{'fraction_non_alphanumeric': 0.2248995983935743, 'fraction_numerical': 0.0011474469305794606, 'mean_word_length': 2.5581632653061224, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6503886', 'n_tokens_mistral': 1310, 'n_tokens_neox': 1157, 'n_words': 263}