<commit_before>(ns clojurewerkz.statistiker.descriptive
  (:require [clojurewerkz.statistiker.fast-math :refer :all]))

(defn get-rank
  [amount percentile]
  (assert (<= percentile 100))
  (-> (* (/ percentile 100) (dec amount))
      Math/ceil
      int))

(def percentiles
  {:min 0
   :max 100
   :median 50
   :25 25
   :75 75})

(defn fivenum
  [values]
  (let [sorted (sort values)
        amount (count values)]
    {:min    (nth sorted (get-rank amount 0))
     :25     (nth sorted (get-rank amount 25))
     :median (nth sorted (get-rank amount 50))
     :75     (nth sorted (get-rank amount 75))
     :max    (nth sorted (get-rank amount 100) )}))<commit_msg>Add IQR, re-do fivenum a bit<commit_after>(ns clojurewerkz.statistiker.descriptive
  (:import [org.apache.commons.math3.stat.descriptive.rank Percentile])
  (:require [clojurewerkz.statistiker.fast-math :refer :all]))

(def percentiles
  {:min 1
   :max 100
   :median 50
   :25 25
   :75 75})

(defn fivenum
  [values]
  (let [p (Percentile.)]
    (.setData p (double-array values))
    (reduce (fn [acc [k v]]
              (assoc acc k (.evaluate p (double v))))
            {}
            percentiles)))

(defn iqr
  [values]
  (let [p              (Percentile.)
        _              (.setData p (double-array values))
        first-quartile (.evaluate p (double 25))
        third-quartile (.evaluate p (double 75))]
    (- third-quartile first-quartile)))
{'fraction_non_alphanumeric': 0.14026517794836008, 'fraction_numerical': 0.03698534542916958, 'mean_word_length': 2.5407407407407407, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4146073', 'n_tokens_mistral': 567, 'n_tokens_neox': 494, 'n_words': 142}