<commit_before>(ns wort.core-test
  (:require [clojure.test :refer :all]
            [wort.core :refer :all]))

(deftest a-test
  (testing "FIXME, I fail."
    (is (= 1 0))))

(deftest splits_on_whitespace
  (is (= ["a" "test"] (get_words_from_key {"key" "a test"} "key")))

  (is (= [] (get_words_from_key {"key" ""} "key")))

  (is (= ["a" "test"] (get_words_from_key {"key" "  a  test  "} "key"))))

(deftest splits_on_punctuation
  (is (= ["a" "test"] (get_words_from_key {"key" " !!!!a...!!!!!test! "} "key")))

  (is (= ["This" "is" "a" "test"] (get_words_from_key {"key" "This, is a test!"} "key")))

  (is (= ["OMG" "Sally" "is" "like" "all" "about" "that" "money" "Maybe" "or" "may" "not" "be" "true"]
         (get_words_from_key {"key" "OMG!!!! Sally, is, like, all about that $money,,... *Maybe or may not be true"} "key"))))<commit_msg>master: just kidding, 1 is 1<commit_after>(ns wort.core-test
  (:require [clojure.test :refer :all]
            [wort.core :refer :all]))

(deftest a-test
  (testing "FIXME, I fail."
    (is (= 1 1))))

(deftest splits_on_whitespace
  (is (= ["a" "test"] (get_words_from_key {"key" "a test"} "key")))

  (is (= [] (get_words_from_key {"key" ""} "key")))

  (is (= ["a" "test"] (get_words_from_key {"key" "  a  test  "} "key"))))

(deftest splits_on_punctuation
  (is (= ["a" "test"] (get_words_from_key {"key" " !!!!a...!!!!!test! "} "key")))

  (is (= ["This" "is" "a" "test"] (get_words_from_key {"key" "This, is a test!"} "key")))

  (is (= ["OMG" "Sally" "is" "like" "all" "about" "that" "money" "Maybe" "or" "may" "not" "be" "true"]
         (get_words_from_key {"key" "OMG!!!! Sally, is, like, all about that $money,,... *Maybe or may not be true"} "key"))))
{'fraction_non_alphanumeric': 0.2545242265032107, 'fraction_numerical': 0.0035026269702276708, 'mean_word_length': 4.011695906432749, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4871315', 'n_tokens_mistral': 747, 'n_tokens_neox': 709, 'n_words': 190}