<commit_before>(ns clojurewerkz.statistiker.metrics-test
  (:require [clojurewerkz.statistiker.metrics :refer :all]
            [clojurewerkz.statistiker.utils :refer [almost=]]
            [clojure.test :refer :all]))
; Tolerance for values
(def tol 1e-8)

(deftest wrong-input-mi
  (testing "invalid inputs for mutual-information"
    (is (thrown? AssertionError (mutual-information [1 2 3] [2 3])))
    (is (thrown? AssertionError (mutual-information [] [2 3])))
    (is (thrown? AssertionError (mutual-information [:b :a :c] [2 :a])))
    (is (thrown? AssertionError (mutual-information [] [])))
    (is (thrown? AssertionError (adjusted_mutual-information [[:b :a] :c] [2 :a])))))

(deftest wrong-input-ami
  (testing "invalid inputs for adjusted_mutual-information"
    (is (thrown? AssertionError (adjusted_mutual-information [1 2 3] [2 3])))
    (is (thrown? AssertionError (adjusted_mutual-information [] [2 3])))
    (is (thrown? AssertionError (adjusted_mutual-information [:b :a :c] [2 :a])))
    (is (thrown? AssertionError (adjusted_mutual-information [[:b :a] :c] [2 :a])))))

(deftest mutual-information-values
  (testing "mutual information calculations"
    (is (= (mutual-information [1 2 3] [1 2 3])
           (mutual-information [3 2 1] [1 2 3])
           (mutual-information [1 2 3] [:a :b 2])))
    (is (almost= (mutual-information [1 2 3] [1 2 3]) 1.0986122886681096 tol))
    (is (= (mutual-information [1 1 1] [1 1 1]) 0.0))
    (is (= (mutual-information [1 2 3 4] [1 1 1 1]) 0.0))
    (is (= (mutual-information [:a :a :a :a] [1 1 1 1]) 0.0))
    (is (almost= (mutual-information [:a 6 :d :f] [1 2 3 4]) 1.3862943611198906 tol))
    (is (almost= (mutual-information [0 0 :c2 :c1] [0 0 87 99]) 1.0397207708399179 tol))
    (is (almost= (mutual-information [0 0 'docid2' 'docid'] [0 99 87 0]) 0.6931471805599453 tol))
    (is (almost= (mutual-information [0 1 2 0] [0 1 2 3]) 1.0397207708399179 tol))))

(deftest adjusted-mutual-information-values
  (testing "adjusted mututal information calculations"
    (is (= (adjusted-mutual-information [1 2 3] [1 2 3])
           (adjusted-mutual-information [3 2 1] [1 2 3])
           (adjusted-mutual-information [1 2 3] [:a :b 2])
           1.0))
  	(is (= (adjusted-mutual-information [] []) 1.0))
    (is (==(adjusted-mutual-information [1 2 3 4] [1 1 1 1]) 0.0))
    (is (= (adjusted-mutual-information [:a :a :a :a] [1 1 1 1]) 1.0))   ;special limit case
    (is (= (adjusted-mutual-information [:a 6 :d :f] [1 2 3 4]) 1.0))
    (is (= (adjusted-mutual-information [0 0 :c2 :c1] [0 0 87 99]) 1.0))
    (is (almost= (adjusted-mutual-information [0 0 'docid2' 'docid'] [0 99 87 0]) -0.20000000000000023 tol))
    (is (= (adjusted-mutual-information [0 1 2 0] [0 1 2 3]) 0.0))))
<commit_msg>Tests rewritten using are, and typo fixed<commit_after>(ns clojurewerkz.statistiker.metrics-test
  (:require [clojurewerkz.statistiker.metrics :refer :all]
            [clojurewerkz.statistiker.utils :refer [almost=]]
            [clojure.test :refer :all]))
; Tolerance for values
(def tol 1e-8)

(deftest wrong-input-mi
  (testing "invalid inputs for mutual-information"
    (are [U V] (thrown? AssertionError (mutual-information U V))
      [1 2 3]       [2 3]
      []            [2 3]
      [:b :a :c]    [2 :a]
      []            []
      [[:b :a] :c]  [2 :a])))

(deftest wrong-input-ami
  (testing "invalid inputs for adjusted-mutual-information"
    (are [U V] (thrown? AssertionError (adjusted-mutual-information U V)) 
      [1 2 3]       [2 3]
      []            [2 3]
      [:b :a :c]    [2 :a]
      [[:b :a] :c]  [2 :a])))

(deftest mutual-information-values
  (testing "mutual information calculations"
    (are [U V s] (almost= (mutual-information U V) s tol)
      [3 2 1]                 [1 2 3]     1.0986122886681096 
      [1 2 3]                 [1 2 3]     1.0986122886681096
      [1 1 1]                 [1 1 1]     0.0
      [1 2 3 4]               [1 1 1 1]   0.0
      [:a :a :a :a]           [1 1 1 1]   0.0
      [:a 6 :d :f]            [1 2 3 4]   1.3862943611198906
      [0 0 :c2 :c1]           [0 0 87 99] 1.0397207708399179
      [0 0 'docid2' 'docid']  [0 99 87 0] 0.6931471805599453
      [0 1 2 0]               [0 1 2 3]   1.0397207708399179)))

(deftest adjusted-mutual-information-values
  (testing "adjusted mututal information calculations"
    (are [U V s] (almost= (adjusted-mutual-information U V) s tol)
      [1 2 3]                 [1 2 3]     1.0
      [3 2 1]                 [1 2 3]     1.0
      [1 2 3]                 [:a :b 2]   1.0
      []                      []          1.0
      [1 2 3 4]               [1 1 1 1]   0.0
      [:a :a :a :a]           [1 1 1 1]   1.0   ;special limit case
      [:a 6 :d :f]            [1 2 3 4]   1.0
      [0 0 :c2 :c1]           [0 0 87 99] 1.0
      [0 0 'docid2' 'docid']  [0 99 87 0] -0.20000000000000023
      [0 1 2 0]               [0 1 2 3]   0.0)))

{'fraction_non_alphanumeric': 0.1588785046728972, 'fraction_numerical': 0.1139780577001219, 'mean_word_length': 2.2538003965631197, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3827907', 'n_tokens_mistral': 2262, 'n_tokens_neox': 1881, 'n_words': 603}