<commit_before>;;; Todo: change, this is probably stupid
(load "trie.lisp")


(defclass word-token ()
  ((token
    :accessor token
    :initarg :token
    :initform nil)
   (data
    :accessor data
    :initarg :data
    :initform nil)))

(defparameter *adjectives* (make-trie-from-file "wordlists/adjectives.txt"))
(defparameter *adverbs* (make-trie-from-file "wordlists/adverbs.txt"))
(defparameter *articles* (make-trie-from-file "wordlists/articles.txt"))
(defparameter *conjugations* (make-trie-from-file "wordlists/conjugations.txt"))
(defparameter *interjections* (make-trie-from-file "wordlists/interjections.txt"))
(defparameter *nouns* (make-trie-from-file "wordlists/nouns.txt"))
(defparameter *prepositions* (make-trie-from-file "wordlists/prepositions.txt"))
(defparameter *pronouns* (make-trie-from-file "wordlists/pronouns.txt"))
(defparameter *verbs* (make-trie-from-file "wordlists/verbs.txt"))

(let ((tries-to-word (list
                      (cons *articles* :articles)
                      (cons *conjugations* :conjugations)
                      (cons *pronouns* :pronouns)
                      (cons *prepositions* :prepositions)
                      (cons *interjections* :interjections)
                      (cons *adverbs* :adverbs)
                      (cons *verbs* :verbs)
                      (cons *adjectives* :adjectives)
                      (cons *nouns* :nouns))))
  (defun tokenize-word (x)
    "Tokenize the word by checking which class it belongs to."
    (let ((tok (cdr (find-if
                     (lambda (a) (trie-member (string-downcase x) (car a)))
                     tries-to-word))))<commit_msg>Better order of word class<commit_after>;;; Todo: change, this is probably stupid
(load "trie.lisp")


(defclass word-token ()
  ((token
    :accessor token
    :initarg :token
    :initform nil)
   (data
    :accessor data
    :initarg :data
    :initform nil)))

(defparameter *adjectives* (make-trie-from-file "wordlists/adjectives.txt"))
(defparameter *adverbs* (make-trie-from-file "wordlists/adverbs.txt"))
(defparameter *articles* (make-trie-from-file "wordlists/articles.txt"))
(defparameter *conjugations* (make-trie-from-file "wordlists/conjugations.txt"))
(defparameter *interjections* (make-trie-from-file "wordlists/interjections.txt"))
(defparameter *nouns* (make-trie-from-file "wordlists/nouns.txt"))
(defparameter *prepositions* (make-trie-from-file "wordlists/prepositions.txt"))
(defparameter *pronouns* (make-trie-from-file "wordlists/pronouns.txt"))
(defparameter *verbs* (make-trie-from-file "wordlists/verbs.txt"))

(let ((tries-to-word (list
                      (cons *articles* :articles)
                      (cons *conjugations* :conjugations)
                      (cons *pronouns* :pronouns)
                      (cons *prepositions* :prepositions)
                      (cons *adverbs* :adverbs)
                      (cons *adjectives* :adjectives)
                      (cons *nouns* :nouns)
                      (cons *interjections* :interjections)
                      (cons *verbs* :verbs))))
  (defun tokenize-word (x)
    "Tokenize the word by checking which class it belongs to."
    (let ((tok (cdr (find-if
                     (lambda (a) (trie-member (string-downcase x) (car a)))
                     tries-to-word))))
{'fraction_non_alphanumeric': 0.14083659343966295, 'fraction_numerical': 0.0, 'mean_word_length': 2.6327868852459018, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2386590', 'n_tokens_mistral': 1118, 'n_tokens_neox': 1064, 'n_words': 235}