<commit_before>(ns language.oneline-summary
  (:require [clojure.core.reducers :as r]))

(def zero-counts
  {:n 0 :s 0.0 :mean 0.0 :m2 0.0})

(defn accum-counts
  ([] zero-counts)
  ([{:keys [n mean m2 s] :as accum} x]
   (let [new-n (inc n)
         delta (- x mean)
         delta-n (/ delta new-n)
         ter])))


<commit_msg>online summary is done<commit_after>(ns language.oneline-summary
  (:require [clojure.core.reducers :as r]))

(def zero-counts
  {:n 0 :s 0.0 :mean 0.0 :m2 0.0})

(defn accum-counts
  ([] zero-counts)
  ([{:keys [n mean m2 s] :as accum} x]
   (let [new-n (inc n)
         delta (- x mean)
         delta-n (/ delta new-n)
         term-1 (* delta delta-n n)
         new-mean (+ mean delta-n)]
     {:n new-n
      :mean new-mean
      :s (+ s x)
      :m2 (+ m2 term-1)})))

(defn op-fields
  "A utility function that calls a function on the values of a field from two maps"
  [op field item1 item2]
  (op (field item1) (field item2)))

(defn combine-counts
  ([] zero-counts)
  ([xa xb]
   (let [n (long (op-fields + :n xa xb))
         delta (op-fields - :mean xb xa)
         nxa*xb (*' (:n xa) (:n xb))]
     {:n n
      :mean (+ (:mean xa) (* delta (/ (:n xb) n)))
      :s (op-fields + :s xa xb)
      :m2 (+ (:m2 xa) (:m2 xb)
             (* delta delta (/ nxa*xb n)))})))

(defn stats-from-sums
  [{:keys [n mean m2 s] :as sums}]
  {:mean (double (/ s n))
   :variance (/ m2 (dec n))})

(defn summary-statistics
  [coll]
  (stats-from-sums
   (r/fold combine-counts accum-counts coll)))

(summary-statistics (repeatedly 10000000 rand))



{'fraction_non_alphanumeric': 0.1798606713109563, 'fraction_numerical': 0.024699176694110196, 'mean_word_length': 2.298538622129436, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1761498', 'n_tokens_mistral': 679, 'n_tokens_neox': 637, 'n_words': 196}