<commit_before>#+clj
(ns jamesmacaulay.zelkova.signal-test
  (:require [jamesmacaulay.async-tools.core :as tools]
            [jamesmacaulay.zelkova.signal :as z]
            [clojure.core.async :as async :refer [go go-loop chan to-chan <! >!]]
            [clojure.core.async.impl.protocols :as impl]
            [jamesmacaulay.async-tools.test :refer (deftest-async)]
            [clojure.test :refer (deftest is are testing)])
  (:import [java.util.Date]))

#+cljs
(ns jamesmacaulay.zelkova.signal-test
  (:require [jamesmacaulay.async-tools.core :as tools]
            [jamesmacaulay.zelkova.signal :as z]
            [cljs.core.async :as async :refer [chan to-chan <! >!]]
            [cljs.core.async.impl.protocols :as impl]
            [jamesmacaulay.async-tools.test :refer-macros (deftest-async)]
            [cemerick.cljs.test :refer-macros (deftest is are testing)])
  (:require-macros [cljs.core.async.macros :refer [go go-loop]]))

(defn event-constructor
  [topic]
  (partial z/->Event topic))

(deftest test-signal-sources
  (let [input (z/input 0)
        foldp (z/foldp + 0 input)
        lift (z/map vector input foldp)
        async (z/async lift)]
    (are [sig sources] (= (z/signal-deps sig) sources)
         input #{}
         foldp #{input}
         lift #{input foldp}
         async #{lift})))

(deftest test-output-node->dependency-map
  (let [input (z/input 0)
        foldp (z/foldp + 0 input)
        lift (z/map vector input foldp)
        async (z/async lift)]
    (are [out deps] (= (z/output-node->dependency-map out) deps)
         input {input #{}}
         foldp {input #{}
                foldp #{input}}
         lift {input #{}
               foldp #{input}
               lift #{input foldp}}
         async {input #{}
                foldp #{input}
                lift #{input foldp}
                async #{lift}})))

(deftest test-topsort
  (let [input (z/input 0)
        foldp (z/foldp + 0 input)
        lift (z/map vector input foldp)
        async (z/async lift)]
    (are [out sorted-sigs] (= (z/topsort out) sorted-sigs)
         input [input]
         foldp [input foldp]
         lift [input foldp lift]
         async [input foldp lift async])))

(deftest-async test-wiring-things-up
  (go
    (let [number (event-constructor :numbers)
          letter (event-constructor :letters)
          numbers-input (z/input 0 :numbers)
          letters-input (z/input :a :letters)
          pairs (z/map vector numbers-input letters-input)
          live-graph (z/spawn pairs)
          output (async/tap live-graph
                            (chan 1 (comp (filter z/fresh?)
                                          (map :value))))]
      (async/onto-chan live-graph
                       [(number 1)
                        (letter :b)
                        (number 2)
                        (letter :c)])
      (is (= [[1 :a] [1 :b] [2 :b] [2 :c]]
             (<! (async/into [] output)))))))

(deftest-async test-io
  (go
    (let [number (event-constructor :numbers)
          in (z/input 0 :numbers)
          graph (z/spawn in)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= 0 (:init in)))
      (async/onto-chan graph (map number [1 2 3]))
      (is (= [1 2 3]
             (<! (async/into [] out)))))))

(deftest-async test-lift
  (go
    (let [number (event-constructor :numbers)
          in (z/input 0 :numbers)
          incremented (z/map inc in)
          graph (z/spawn incremented)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= 1 (:init incremented)))
      (async/onto-chan graph (map number [1 2 3]))
      (is (= [2 3 4]
             (<! (async/into [] out)))))
    (let [[a b c] (map event-constructor [:a :b :c])
          ins (map (partial z/input 0) [:a :b :c])
          summed (apply z/map + ins)
          graph (z/spawn summed)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= 0 (:init summed)))
      (async/onto-chan graph [(a 1) (b 2) (c 3) (a 10)])
      (is (= [1 3 6 15]
             (<! (async/into [] out)))))
    (let [zero-arity-+-lift (z/map +)
          zero-arity-vector-lift (z/map vector)]
      (is (= 0 (:init zero-arity-+-lift)))
      (is (= [] (:init zero-arity-vector-lift))))))

(deftest-async test-foldp
  (go
    (let [number (event-constructor :numbers)
          in (z/input 0 :numbers)
          sum (z/foldp + 0 in)
          graph (z/spawn sum)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= 0 (:init sum)))
      (async/onto-chan graph (map number [1 2 3]))
      (is (= [1 3 6]
             (<! (async/into [] out)))))))

(deftest-async test-regular-signals-are-synchronous
  (go
    (let [number (event-constructor :numbers)
          in (z/input 0 :numbers)
          decremented (z/map dec in)
          incremented (z/map inc in)
          combined (z/map (fn [a b] {:decremented a
                                            :incremented b})
                                 decremented
                                 incremented)
          graph (z/spawn combined)
          out (async/tap graph (chan 1 z/fresh-values))]
      (async/onto-chan graph (map number [2 10]))
      (is (= [{:decremented 1
               :incremented 3}
              {:decremented 9
               :incremented 11}]
             (<! (async/into [] out)))))))

(deftest-async test-constant
  (go
    (let [number (event-constructor :numbers)
          in (z/input 0 :numbers)
          foo (z/constant :foo)
          combined (z/map vector in foo)
          graph (z/spawn combined)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= [0 :foo] (:init combined)))
      (async/onto-chan graph (map number [1 2 3]))
      (is (= [[1 :foo] [2 :foo] [3 :foo]]
             (<! (async/into [] out)))))))

(deftest-async test-merge
  (go
    (let [a (event-constructor :a)
          b (event-constructor :b)
          a-in (z/input 10 :a)
          b-in (z/input 20 :b)
          b-dec (z/map dec b-in)
          b-inc (z/map inc b-in)
          merged (z/merge a-in b-dec b-in b-inc)
          graph (z/spawn merged)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= 10 (:init merged)))
      (async/onto-chan graph [(a 20) (b 30) (a 40) (b 50)])
      (is (= [20 29 40 49]
             (<! (async/into [] out)))))))

(deftest-async test-combine
  (go
    (let [number (event-constructor :numbers)
          in (z/input 0 :numbers)
          inc'd (z/map inc in)
          combined (z/combine [in inc'd])
          graph (z/spawn combined)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= [0 1] (:init combined)))
      (async/onto-chan graph (map number [1 2]))
      (is (= [[1 2] [2 3]]
             (<! (async/into [] out)))))
    (let [empty-combined (z/combine [])]
      (is (= [] (:init empty-combined))))))


(deftest-async test-sample-on
  (go
    (let [pos (event-constructor :mouse-position)
          click ((event-constructor :mouse-clicks) :click)
          fake-mouse-position (z/input [0 0] :mouse-position)
          fake-mouse-clicks (z/input :click :mouse-clicks)
          sampled (z/sample-on fake-mouse-clicks fake-mouse-position)
          graph (z/spawn sampled)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= [0 0] (:init sampled)))
      (async/onto-chan graph
                       [(pos [10 10])
                        click
                        (pos [20 20])
                        (pos [30 30])
                        click
                        (pos [40 40])
                        (pos [50 50])
                        click])
      (is (= [[10 10] [30 30] [50 50]]
             (<! (async/into [] out)))))))

(deftest-async test-transducep
  (go
    (let [number (event-constructor :numbers)
          in (z/input 10 :numbers)
          odd-increments (z/transducep (comp (map inc)
                                                   (filter odd?))
                                             conj
                                             in)
          graph (z/spawn odd-increments)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= [] (:init odd-increments)))
      (async/onto-chan graph (map number [20 21 22 23]))
      (is (= [[21]
              [21 23]]
             (<! (async/into [] out)))))))

(deftest-async test-count
  (go
    (let [in1-event (event-constructor :in1)
          in2-event (event-constructor :in2)
          in1 (z/input 1 :in1)
          in2 (z/input 1 :in2)
          count1 (z/count in1)
          combined (z/map vector count1 in1 in2)
          graph (z/spawn combined)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= 0 (:init count1)))
      (is (= [0 1 1] (:init combined)))
      (async/onto-chan graph [(in1-event 2)
                              (in1-event 3)
                              (in2-event 2)
                              (in1-event 4)])
      (is (= [[1 2 1]
              [2 3 1]
              [2 3 2]
              [3 4 2]]
             (<! (async/into [] out)))))))

(deftest-async test-count-if
  (go
    (let [in1-event (event-constructor :in1)
          in2-event (event-constructor :in2)
          in1 (z/input 1 :in1)
          in2 (z/input 1 :in2)
          count1-odd (z/count-if odd? in1)
          combined (z/map vector count1-odd in1 in2)
          graph (z/spawn combined)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= 0 (:init count1-odd)))
      (async/onto-chan graph [(in1-event 2)
                              (in1-event 3)
                              (in2-event 2)
                              (in1-event 4)
                              (in1-event 5)])
      (is (= [[0 2 1]
              [1 3 1]
              [1 3 2]
              [1 4 2]
              [2 5 2]]
             (<! (async/into [] out)))))))

(deftest-async test-keep-if
  (go
    (let [number (event-constructor :numbers)
          in (z/input 0 :numbers)
          oddnums (z/keep-if odd? -1 in)
          count-odd (z/count oddnums)
          evennums (z/keep-if even? -2 in)
          count-even (z/count evennums)
          combined (z/map vector oddnums count-odd evennums count-even)
          graph (z/spawn combined)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= [-1 0 0 0] (:init combined)))
      (async/onto-chan graph (map number [1 2 3]))
      (is (= [[1 1 0 0]
              [1 1 2 1]
              [3 2 2 1]]
             (<! (async/into [] out)))))))

(deftest-async test-keep-when
  (go
    (let [number (event-constructor :numbers)
          letter (event-constructor :letters)
          numbers-in (z/input 0 :numbers)
          letters-in (z/input :a :letters)
          odd-kept-letters (z/keep-when (z/map odd? numbers-in) :false-init letters-in)
          graph (z/spawn odd-kept-letters)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= :false-init (:init odd-kept-letters)))
      (is (= :a (:init (z/keep-when (z/map even? numbers-in) :z letters-in))))
      (async/onto-chan graph [(letter :b)
                              (number 1)
                              (letter :c)
                              (number 2)
                              (letter :d)
                              (letter :e)
                              (number 3)
                              (letter :f)])
      (is (= [:c :f] (<! (async/into [] out)))))))

(deftest-async test-drop-repeats
  (go
    (let [number (event-constructor :numbers)
          in (z/input 0 :numbers)
          no-repeats (z/drop-repeats in)
          graph (z/spawn no-repeats)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= 0 (:init no-repeats)))
      (async/onto-chan graph (map number [1 1 2 1 2 2 2 1 1]))
      (is (= [1 2 1 2 1] (<! (async/into [] out)))))))

(deftest-async test-world-building
  (go
    (let [pos (event-constructor :mouse-position)
          event-source (async/to-chan (map pos [[10 10]
                                                [20 20]
                                                [30 30]]))
          mouse-position (assoc (z/input [0 0] :mouse-position)
                           :event-sources {:mouse-position (constantly event-source)})
          graph (z/spawn mouse-position)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= [[10 10] [20 20] [30 30]]
             (<! (async/into [] out)))))))

(deftest-async test-async-makes-signals-asynchronous
  (go
    (let [number (event-constructor :numbers)
          in (z/input 0 :numbers)
          decremented (z/map dec in)
          incremented (z/map inc in)
          async-incremented (z/async incremented)
          combined (z/combine [decremented async-incremented])<commit_msg>actually use public interface in test<commit_after>#+clj
(ns jamesmacaulay.zelkova.signal-test
  (:require [jamesmacaulay.async-tools.core :as tools]
            [jamesmacaulay.zelkova.signal :as z]
            [clojure.core.async :as async :refer [go go-loop chan to-chan <! >!]]
            [clojure.core.async.impl.protocols :as impl]
            [jamesmacaulay.async-tools.test :refer (deftest-async)]
            [clojure.test :refer (deftest is are testing)])
  (:import [java.util.Date]))

#+cljs
(ns jamesmacaulay.zelkova.signal-test
  (:require [jamesmacaulay.async-tools.core :as tools]
            [jamesmacaulay.zelkova.signal :as z]
            [cljs.core.async :as async :refer [chan to-chan <! >!]]
            [cljs.core.async.impl.protocols :as impl]
            [jamesmacaulay.async-tools.test :refer-macros (deftest-async)]
            [cemerick.cljs.test :refer-macros (deftest is are testing)])
  (:require-macros [cljs.core.async.macros :refer [go go-loop]]))

(defn event-constructor
  [topic]
  (partial z/->Event topic))

(deftest test-signal-sources
  (let [input (z/input 0)
        foldp (z/foldp + 0 input)
        lift (z/map vector input foldp)
        async (z/async lift)]
    (are [sig sources] (= (z/signal-deps sig) sources)
         input #{}
         foldp #{input}
         lift #{input foldp}
         async #{lift})))

(deftest test-output-node->dependency-map
  (let [input (z/input 0)
        foldp (z/foldp + 0 input)
        lift (z/map vector input foldp)
        async (z/async lift)]
    (are [out deps] (= (z/output-node->dependency-map out) deps)
         input {input #{}}
         foldp {input #{}
                foldp #{input}}
         lift {input #{}
               foldp #{input}
               lift #{input foldp}}
         async {input #{}
                foldp #{input}
                lift #{input foldp}
                async #{lift}})))

(deftest test-topsort
  (let [input (z/input 0)
        foldp (z/foldp + 0 input)
        lift (z/map vector input foldp)
        async (z/async lift)]
    (are [out sorted-sigs] (= (z/topsort out) sorted-sigs)
         input [input]
         foldp [input foldp]
         lift [input foldp lift]
         async [input foldp lift async])))

(deftest-async test-wiring-things-up
  (go
    (let [number (event-constructor :numbers)
          letter (event-constructor :letters)
          numbers-input (z/input 0 :numbers)
          letters-input (z/input :a :letters)
          pairs (z/map vector numbers-input letters-input)
          live-graph (z/spawn pairs)
          output (async/tap live-graph
                            (chan 1 (comp (filter z/fresh?)
                                          (map :value))))]
      (async/onto-chan live-graph
                       [(number 1)
                        (letter :b)
                        (number 2)
                        (letter :c)])
      (is (= [[1 :a] [1 :b] [2 :b] [2 :c]]
             (<! (async/into [] output)))))))

(deftest-async test-io
  (go
    (let [number (event-constructor :numbers)
          in (z/input 0 :numbers)
          graph (z/spawn in)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= 0 (:init in)))
      (async/onto-chan graph (map number [1 2 3]))
      (is (= [1 2 3]
             (<! (async/into [] out)))))))

(deftest-async test-lift
  (go
    (let [number (event-constructor :numbers)
          in (z/input 0 :numbers)
          incremented (z/map inc in)
          graph (z/spawn incremented)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= 1 (:init incremented)))
      (async/onto-chan graph (map number [1 2 3]))
      (is (= [2 3 4]
             (<! (async/into [] out)))))
    (let [[a b c] (map event-constructor [:a :b :c])
          ins (map (partial z/input 0) [:a :b :c])
          summed (apply z/map + ins)
          graph (z/spawn summed)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= 0 (:init summed)))
      (async/onto-chan graph [(a 1) (b 2) (c 3) (a 10)])
      (is (= [1 3 6 15]
             (<! (async/into [] out)))))
    (let [zero-arity-+-lift (z/map +)
          zero-arity-vector-lift (z/map vector)]
      (is (= 0 (:init zero-arity-+-lift)))
      (is (= [] (:init zero-arity-vector-lift))))))

(deftest-async test-foldp
  (go
    (let [number (event-constructor :numbers)
          in (z/input 0 :numbers)
          sum (z/foldp + 0 in)
          graph (z/spawn sum)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= 0 (:init sum)))
      (async/onto-chan graph (map number [1 2 3]))
      (is (= [1 3 6]
             (<! (async/into [] out)))))))

(deftest-async test-regular-signals-are-synchronous
  (go
    (let [number (event-constructor :numbers)
          in (z/input 0 :numbers)
          decremented (z/map dec in)
          incremented (z/map inc in)
          combined (z/map (fn [a b] {:decremented a
                                            :incremented b})
                                 decremented
                                 incremented)
          graph (z/spawn combined)
          out (async/tap graph (chan 1 z/fresh-values))]
      (async/onto-chan graph (map number [2 10]))
      (is (= [{:decremented 1
               :incremented 3}
              {:decremented 9
               :incremented 11}]
             (<! (async/into [] out)))))))

(deftest-async test-constant
  (go
    (let [number (event-constructor :numbers)
          in (z/input 0 :numbers)
          foo (z/constant :foo)
          combined (z/map vector in foo)
          graph (z/spawn combined)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= [0 :foo] (:init combined)))
      (async/onto-chan graph (map number [1 2 3]))
      (is (= [[1 :foo] [2 :foo] [3 :foo]]
             (<! (async/into [] out)))))))

(deftest-async test-merge
  (go
    (let [a (event-constructor :a)
          b (event-constructor :b)
          a-in (z/input 10 :a)
          b-in (z/input 20 :b)
          b-dec (z/map dec b-in)
          b-inc (z/map inc b-in)
          merged (z/merge a-in b-dec b-in b-inc)
          graph (z/spawn merged)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= 10 (:init merged)))
      (async/onto-chan graph [(a 20) (b 30) (a 40) (b 50)])
      (is (= [20 29 40 49]
             (<! (async/into [] out)))))))

(deftest-async test-combine
  (go
    (let [number (event-constructor :numbers)
          in (z/input 0 :numbers)
          inc'd (z/map inc in)
          combined (z/combine [in inc'd])
          graph (z/spawn combined)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= [0 1] (:init combined)))
      (async/onto-chan graph (map number [1 2]))
      (is (= [[1 2] [2 3]]
             (<! (async/into [] out)))))
    (let [empty-combined (z/combine [])]
      (is (= [] (:init empty-combined))))))


(deftest-async test-sample-on
  (go
    (let [pos (event-constructor :mouse-position)
          click ((event-constructor :mouse-clicks) :click)
          fake-mouse-position (z/input [0 0] :mouse-position)
          fake-mouse-clicks (z/input :click :mouse-clicks)
          sampled (z/sample-on fake-mouse-clicks fake-mouse-position)
          graph (z/spawn sampled)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= [0 0] (:init sampled)))
      (async/onto-chan graph
                       [(pos [10 10])
                        click
                        (pos [20 20])
                        (pos [30 30])
                        click
                        (pos [40 40])
                        (pos [50 50])
                        click])
      (is (= [[10 10] [30 30] [50 50]]
             (<! (async/into [] out)))))))

(deftest-async test-transducep
  (go
    (let [number (event-constructor :numbers)
          in (z/input 10 :numbers)
          odd-increments (z/transducep (comp (map inc)
                                                   (filter odd?))
                                             conj
                                             in)
          graph (z/spawn odd-increments)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= [] (:init odd-increments)))
      (async/onto-chan graph (map number [20 21 22 23]))
      (is (= [[21]
              [21 23]]
             (<! (async/into [] out)))))))

(deftest-async test-count
  (go
    (let [in1-event (event-constructor :in1)
          in2-event (event-constructor :in2)
          in1 (z/input 1 :in1)
          in2 (z/input 1 :in2)
          count1 (z/count in1)
          combined (z/map vector count1 in1 in2)
          graph (z/spawn combined)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= 0 (:init count1)))
      (is (= [0 1 1] (:init combined)))
      (async/onto-chan graph [(in1-event 2)
                              (in1-event 3)
                              (in2-event 2)
                              (in1-event 4)])
      (is (= [[1 2 1]
              [2 3 1]
              [2 3 2]
              [3 4 2]]
             (<! (async/into [] out)))))))

(deftest-async test-count-if
  (go
    (let [in1-event (event-constructor :in1)
          in2-event (event-constructor :in2)
          in1 (z/input 1 :in1)
          in2 (z/input 1 :in2)
          count1-odd (z/count-if odd? in1)
          combined (z/map vector count1-odd in1 in2)
          graph (z/spawn combined)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= 0 (:init count1-odd)))
      (async/onto-chan graph [(in1-event 2)
                              (in1-event 3)
                              (in2-event 2)
                              (in1-event 4)
                              (in1-event 5)])
      (is (= [[0 2 1]
              [1 3 1]
              [1 3 2]
              [1 4 2]
              [2 5 2]]
             (<! (async/into [] out)))))))

(deftest-async test-keep-if
  (go
    (let [number (event-constructor :numbers)
          in (z/input 0 :numbers)
          oddnums (z/keep-if odd? -1 in)
          count-odd (z/count oddnums)
          evennums (z/keep-if even? -2 in)
          count-even (z/count evennums)
          combined (z/map vector oddnums count-odd evennums count-even)
          graph (z/spawn combined)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= [-1 0 0 0] (:init combined)))
      (async/onto-chan graph (map number [1 2 3]))
      (is (= [[1 1 0 0]
              [1 1 2 1]
              [3 2 2 1]]
             (<! (async/into [] out)))))))

(deftest-async test-keep-when
  (go
    (let [number (event-constructor :numbers)
          letter (event-constructor :letters)
          numbers-in (z/input 0 :numbers)
          letters-in (z/input :a :letters)
          odd-kept-letters (z/keep-when (z/map odd? numbers-in) :false-init letters-in)
          graph (z/spawn odd-kept-letters)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= :false-init (:init odd-kept-letters)))
      (is (= :a (:init (z/keep-when (z/map even? numbers-in) :z letters-in))))
      (async/onto-chan graph [(letter :b)
                              (number 1)
                              (letter :c)
                              (number 2)
                              (letter :d)
                              (letter :e)
                              (number 3)
                              (letter :f)])
      (is (= [:c :f] (<! (async/into [] out)))))))

(deftest-async test-drop-repeats
  (go
    (let [number (event-constructor :numbers)
          in (z/input 0 :numbers)
          no-repeats (z/drop-repeats in)
          graph (z/spawn no-repeats)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= 0 (:init no-repeats)))
      (async/onto-chan graph (map number [1 1 2 1 2 2 2 1 1]))
      (is (= [1 2 1 2 1] (<! (async/into [] out)))))))

(deftest-async test-world-building
  (go
    (let [event-source (async/to-chan [[10 10]
                                       [20 20]
                                       [30 30]])
          mouse-position (z/input [0 0] :mouse-position (constantly event-source))
          graph (z/spawn mouse-position)
          out (async/tap graph (chan 1 z/fresh-values))]
      (is (= [[10 10] [20 20] [30 30]]
             (<! (async/into [] out)))))))

(deftest-async test-async-makes-signals-asynchronous
  (go
    (let [number (event-constructor :numbers)
          in (z/input 0 :numbers)
          decremented (z/map dec in)
          incremented (z/map inc in)
          async-incremented (z/async incremented)
          combined (z/combine [decremented async-incremented])
{'fraction_non_alphanumeric': 0.1472877358490566, 'fraction_numerical': 0.02507861635220126, 'mean_word_length': 1.5286750819998012, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 31, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '522577', 'n_tokens_mistral': 8916, 'n_tokens_neox': 8091, 'n_words': 2320}