<commit_before>(ns org.kanbanik.statistics-test
  (:use clojure.test)
  (:use org.kanbanik.statistics))

(testing "Statistics"
  (testing "first-timestamp"
    (is (= -1 (first-timestamp [])))
    (is (= -1 (first-timestamp [{:does-not-have-timestamp 10}])))
    (is (= 9 (first-timestamp [{:timestamp 10}])))
    (is (= 9 (first-timestamp [{:timestamp 10} {:timestamp 12}])))
    )

  (testing "reduce-tasks"
     (let [
          id1t10 {:timestamp 10 :id 1}
          id1t20 {:timestamp 20 :id 1}
          id2t30 {:timestamp 30 :id 2}
         ]
; one chunk, one field in it
      (is (= 9 (:timestamp (first (first (reduce-tasks [[1 [id1t10]]] 1))))))
; one chunk, two field with same id in it
      (is (= 19 (:timestamp (first (first (reduce-tasks [[1 [id1t10 id1t20]]] 1))))))
; one chunk, 3 fields with 2 differend ids in it
      (let [two-in-chunk (reduce-tasks [[1 [id1t10 id1t20 id2t30]]] 1)]
        (is (and
             (= 19 (:timestamp (first (first two-in-chunk))))
             (= 29 (:timestamp (second (first two-in-chunk))))
             ))
        )
; two chunks and 2 differnet ids in both
      (let [two-chunks (reduce-tasks [[1 [id1t10 id1t20 id2t30]] [2 [id2t30 id1t20 id1t10]]] 1)]
        (is (and
; test first chunk
             (= 19 (:timestamp (first (first two-chunks))))
             (= 29 (:timestamp (second (first two-chunks))))
; test second chunk
             (= 29 (:timestamp (first (second two-chunks))))
             (= 9 (:timestamp (second (second two-chunks))))
             ))
        )
      )
  )

  (testing "group-by-timeframe"
    (let [
          id1t10 {:timestamp 10 :id 1}
          id1t20 {:timestamp 20 :id 1}
          id2t30 {:timestamp 30 :id 2}
         ]
      (is (= 0 (count (group-by-timeframe [] 1))))
      (is (= 3 (count (group-by-timeframe [id1t10 id1t20 id2t30] 10))))
      (is (= 3 (count (group-by-timeframe [id1t10 id1t20 id2t30] 1))))
      (is (= 3 (count (group-by-timeframe [id1t10 id1t20 id2t30] 0))))
      )
    )

  (testing "apply-filter"
    (let [
          w1 {:workflowitem-id 1}
          w2 {:workflowitem-id 2}
          w3 {:workflowitem-id 3}
          w4 {:workflowitem-id 4}
          ]
    (is (= 0 (count (apply-filter [] []))))
    (is (= 1 (count (apply-filter w1 [w2 w1]))))
    (is (= 0 (count (apply-filter {} [w2 w1]))))
    (is (= 0 (count (apply-filter {} [{} w1]))))
    )
  )
)<commit_msg>Added tests for the apply-function<commit_after>(ns org.kanbanik.statistics-test
  (:use clojure.test)
  (:use org.kanbanik.statistics))

(testing "Statistics"
  (testing "first-timestamp"
    (is (= -1 (first-timestamp [])))
    (is (= -1 (first-timestamp [{:does-not-have-timestamp 10}])))
    (is (= 9 (first-timestamp [{:timestamp 10}])))
    (is (= 9 (first-timestamp [{:timestamp 10} {:timestamp 12}])))
    )

  (testing "reduce-tasks"
     (let [
          id1t10 {:timestamp 10 :id 1}
          id1t20 {:timestamp 20 :id 1}
          id2t30 {:timestamp 30 :id 2}
         ]
; one chunk, one field in it
      (is (= 9 (:timestamp (first (first (reduce-tasks [[1 [id1t10]]] 1))))))
; one chunk, two field with same id in it
      (is (= 19 (:timestamp (first (first (reduce-tasks [[1 [id1t10 id1t20]]] 1))))))
; one chunk, 3 fields with 2 differend ids in it
      (let [two-in-chunk (reduce-tasks [[1 [id1t10 id1t20 id2t30]]] 1)]
        (is (and
             (= 19 (:timestamp (first (first two-in-chunk))))
             (= 29 (:timestamp (second (first two-in-chunk))))
             ))
        )
; two chunks and 2 differnet ids in both
      (let [two-chunks (reduce-tasks [[1 [id1t10 id1t20 id2t30]] [2 [id2t30 id1t20 id1t10]]] 1)]
        (is (and
; test first chunk
             (= 19 (:timestamp (first (first two-chunks))))
             (= 29 (:timestamp (second (first two-chunks))))
; test second chunk
             (= 29 (:timestamp (first (second two-chunks))))
             (= 9 (:timestamp (second (second two-chunks))))
             ))
        )
      )
  )

  (testing "group-by-timeframe"
    (let [
          id1t10 {:timestamp 10 :id 1}
          id1t20 {:timestamp 20 :id 1}
          id2t30 {:timestamp 30 :id 2}
         ]
      (is (= 0 (count (group-by-timeframe [] 1))))
      (is (= 3 (count (group-by-timeframe [id1t10 id1t20 id2t30] 10))))
      (is (= 3 (count (group-by-timeframe [id1t10 id1t20 id2t30] 1))))
      (is (= 3 (count (group-by-timeframe [id1t10 id1t20 id2t30] 0))))
      )
    )

  (testing "apply-filter"
    (let [
          w1 {:workflowitem-id 1}
          w2 {:workflowitem-id 2}
          w3 {:workflowitem-id 3}
          w4 {:workflowitem-id 4}
          ]
    (is (= 0 (count (apply-filter [] []))))
    (is (= 1 (count (apply-filter w1 [w2 w1]))))
    (is (= 0 (count (apply-filter {} [w2 w1]))))
    (is (= 0 (count (apply-filter {} [{} w1]))))
    )
  )

  (testing "apply-function" 
    (is (= 0 (apply-function :cnt [])))
    (is (= 1 (apply-function :cnt [:x])))
    (is (= nil (apply-function :not-existing [])))
    (is (= nil (apply-function nil [])))
  )
)
{'fraction_non_alphanumeric': 0.17677470670113343, 'fraction_numerical': 0.06561940743686617, 'mean_word_length': 2.2388924661944625, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4442383', 'n_tokens_mistral': 2025, 'n_tokens_neox': 1753, 'n_words': 511}