<commit_before>* Namespace: thi.ng.data
** thi.ng.data.core
*** Namespace declaration
#+BEGIN_SRC clojure :tangle babel/src-cljx/thi/ng/data/core.cljx
  (ns thi.ng.data.core)
#+END_SRC
*** Indexing functions
#+BEGIN_SRC clojure :tangle babel/src-cljx/thi/ng/data/core.cljx
  (defn index
    [i t]
    (if-let [tt (get i t)] [i tt] [(conj i t) t]))

  (defn index!
    [i t]
    (if-let [tt (get i t)] [i tt] [(conj! i t) t]))

  (defn index-kv
    [m k v]
    (if-let [[k vv] (find m k)]
      (assoc m k (conj vv v))
      (assoc m k #{v})))

  (defn index-kv!
    [m k v]
    (if-let [vv (get m k)]
      (assoc! m k (conj vv v))
      (assoc! m k #{v})))

  (defn value-set
    ([idx v] (into #{} (get idx v)))
    ([f idx v] (persistent! (reduce #(conj! %1 (f %2)) (transient #{}) (get idx v)))))
#+END_SRC
*** Sequence functions
#+BEGIN_SRC clojure :tangle babel/src-cljx/thi/ng/data/core.cljx
  (defn bisect
    ([f coll]
       (let [[m n] (reduce
                    (fn [[m n] v] (if (f v) [(conj! m v) n] [m (conj! n v)]))
                    [(transient []) (transient [])] coll)]
         [(persistent! m) (persistent! n)]))
    ([f f2 coll]
       (let [[m n] (bisect f coll)]
         [(f2 m) (f2 n)])))

  (defn neighbors
    [x coll]
    (let [n (dec (count coll))]
      (loop [i n]
        (when (>= i 0)
          (if (= x (coll i))
            [(coll (if (pos? i) (dec i) n))
             (coll (if (< i n) (inc i) 0))]
            (recur (dec i)))))))

  (defn successive-nth
    "Returns a lazyseq of `n`-element vectors, each one containing
    a successive elements of the original collection.

        (successive-nth 3 [1 2 3 4])
        => ([1 2 3] [2 3 4] [3 4 5])"
    [n coll]
    (lazy-seq
     (let [s (take n coll)]
       (if (= n (count s))
         (cons (vec s) (successive-nth n (rest coll)))))))

  (defn successive-nth-indexed
    "Returns a lazyseq of nested 2-element vectors, each one containing
    a vector of `n` successive elements of the original collection and
    an sequence index.

        (successive-nth-indexed 2 [10 20 30 40])
        => ([[10 20] 0] [[20 30] 1] [[30 40] 2])"
    ([n coll] (successive-nth-indexed n 0 coll))
    ([n idx coll]
       (lazy-seq
        (let [s (take n coll)]
          (if (= n (count s))
            (cons [(vec s) idx]
                  (successive-nth-indexed n (inc idx) (rest coll))))))))

  (defn apply-to-keys
    "Applies `f` with `args` to all given `keys` in `type`."
    [type keys f & args]
    (reduce (fn[acc k] (assoc acc k (apply f (get type k) args))) type keys))

  (defn reduce-pairs
    [f1 f2 coll]
    (when (> (count coll) 1)
      (reduce f1 (map (fn [[a b]] (f2 a b))
                      (successive-nth 2 coll)))))

  (defn wrap-seq
    [s head tail]
    (concat
     (if (sequential? head) (concat head s) (cons head s))
     (if (sequential? tail) tail [tail])))

  (defn all-after
    "Returns a new collection of all items after `item` in original `coll`.
    If `coll` is a vector, the new collection is created with `subvec`.
    Returns original coll if item isn't found."
    [item coll]
    (let [idx (inc (.indexOf coll item))]
      (if (pos? idx)
        (if (vector? coll) (subvec coll idx) (drop idx coll))
        coll)))

  (defn iterate-while
    [pred f x]
    (lazy-seq
     (when (pred x)
       (cons x (iterate-while pred f (f x))))))
#+END_SRC
** thi.ng.data.byteutils
*** TODO refactor to be CLJS compatible
*** Namespace declaration
#+BEGIN_SRC clojure :tangle babel/src-cljx/thi/ng/data/byteutils.cljx
  (ns thi.ng.data.byteutils)
#+END_SRC
*** Conversion functions
#+BEGIN_SRC clojure :tangle babel/src-cljx/thi/ng/data/byteutils.cljx
  (defn int->byte [x] (if (> x 0x7f) (- x 0x100) x))

  (defn byte->int [x] (if (neg? x) (+ x 0x100) x))
#+END_SRC
*** Write functions
#+BEGIN_SRC clojure :tangle babel/src-cljx/thi/ng/data/byteutils.cljx
  (defn write-str-bytes
    [out x]
    (.write out (.getBytes x))
    out)

  (defn write-int16-le
    [out x]
    (.write
     out
     (byte-array
      [(unchecked-byte (bit-and x 0xff))
       (unchecked-byte (bit-and (bit-shift-right x 8) 0xff))]))
    out)

  (defn write-int32-le
    [out x]
    (.write
     out
     (byte-array
      [(unchecked-byte (bit-and x 0xff))
       (unchecked-byte (bit-and (bit-shift-right x 8) 0xff))
       (unchecked-byte (bit-and (bit-shift-right x 16) 0xff))
       (unchecked-byte (bit-shift-right x 24))]))
    out)

  (defn write-float-le
    [out x]
    (write-int32-le out (Float/floatToRawIntBits x)))

  (defn write-vec3-le
    [out [x y z]]
    (write-float-le out x)
    (write-float-le out y)
    (write-float-le out z))
#+END_SRC
*** Read functions
#+BEGIN_SRC clojure :tangle babel/src-cljx/thi/ng/data/byteutils.cljx
  (defn read-int16-le
    [in]
    (let [buf (byte-array 2)]
      (.read in buf 0 2)
      (bit-or (byte->int (aget buf 0)) (bit-shift-left (byte->int (aget buf 1)) 8))))

  (defn read-int32-le
    [in]
    (let [buf (byte-array 4)]
      (.read in buf 0 4)
      (bit-or
       (byte->int (aget buf 0))
       (bit-shift-left (byte->int (aget buf 1)) 8)
       (bit-shift-left (byte->int (aget buf 2)) 16)
       (bit-shift-left (byte->int (aget buf 3)) 24))))

  (defn read-float-le
    [in]
    (Float/intBitsToFloat (unchecked-int (read-int32-le in))))

  (defn read-vec3-le
    [in]
    [(read-float-le in) (read-float-le in) (read-float-le in)])
#+END_SRC<commit_msg>adding data.core/shift-left fn, adding type hints to byteutils<commit_after>* Namespace: thi.ng.data
** thi.ng.data.core
*** Namespace declaration
#+BEGIN_SRC clojure :tangle babel/src-cljx/thi/ng/data/core.cljx
  (ns thi.ng.data.core)
#+END_SRC
*** Indexing functions
#+BEGIN_SRC clojure :tangle babel/src-cljx/thi/ng/data/core.cljx
  (defn index
    [i t]
    (if-let [tt (get i t)] [i tt] [(conj i t) t]))

  (defn index!
    [i t]
    (if-let [tt (get i t)] [i tt] [(conj! i t) t]))

  (defn index-kv
    [m k v]
    (if-let [[k vv] (find m k)]
      (assoc m k (conj vv v))
      (assoc m k #{v})))

  (defn index-kv!
    [m k v]
    (if-let [vv (get m k)]
      (assoc! m k (conj vv v))
      (assoc! m k #{v})))

  (defn value-set
    ([idx v] (into #{} (get idx v)))
    ([f idx v] (persistent! (reduce #(conj! %1 (f %2)) (transient #{}) (get idx v)))))
#+END_SRC
*** Sequence functions
#+BEGIN_SRC clojure :tangle babel/src-cljx/thi/ng/data/core.cljx
  (defn bisect
    ([f coll]
       (let [[m n] (reduce
                    (fn [[m n] v] (if (f v) [(conj! m v) n] [m (conj! n v)]))
                    [(transient []) (transient [])] coll)]
         [(persistent! m) (persistent! n)]))
    ([f f2 coll]
       (let [[m n] (bisect f coll)]
         [(f2 m) (f2 n)])))

  (defn neighbors
    [x coll]
    (let [n (dec (count coll))]
      (loop [i n]
        (when (>= i 0)
          (if (= x (coll i))
            [(coll (if (pos? i) (dec i) n))
             (coll (if (< i n) (inc i) 0))]
            (recur (dec i)))))))

  (defn successive-nth
    "Returns a lazyseq of `n`-element vectors, each one containing
    a successive elements of the original collection.

        (successive-nth 3 [1 2 3 4])
        => ([1 2 3] [2 3 4] [3 4 5])"
    [n coll]
    (lazy-seq
     (let [s (take n coll)]
       (if (= n (count s))
         (cons (vec s) (successive-nth n (rest coll)))))))

  (defn successive-nth-indexed
    "Returns a lazyseq of nested 2-element vectors, each one containing
    a vector of `n` successive elements of the original collection and
    an sequence index.

        (successive-nth-indexed 2 [10 20 30 40])
        => ([[10 20] 0] [[20 30] 1] [[30 40] 2])"
    ([n coll] (successive-nth-indexed n 0 coll))
    ([n idx coll]
       (lazy-seq
        (let [s (take n coll)]
          (if (= n (count s))
            (cons [(vec s) idx]
                  (successive-nth-indexed n (inc idx) (rest coll))))))))

  (defn apply-to-keys
    "Applies `f` with `args` to all given `keys` in `type`."
    [type keys f & args]
    (reduce (fn[acc k] (assoc acc k (apply f (get type k) args))) type keys))

  (defn reduce-pairs
    [f1 f2 coll]
    (when (> (count coll) 1)
      (reduce f1 (map (fn [[a b]] (f2 a b))
                      (successive-nth 2 coll)))))

  (defn wrap-seq
    [s head tail]
    (concat
     (if (sequential? head) (concat head s) (cons head s))
     (if (sequential? tail) tail [tail])))

  (defn shift-left
    [n xs]
    (concat (drop n xs) (take n xs)))

  (defn all-after
    "Returns a new collection of all items after `item` in original `coll`.
    If `coll` is a vector, the new collection is created with `subvec`.
    Returns original coll if item isn't found."
    [item coll]
    (let [idx (inc (.indexOf coll item))]
      (if (pos? idx)
        (if (vector? coll) (subvec coll idx) (drop idx coll))
        coll)))

  (defn iterate-while
    [pred f x]
    (lazy-seq
     (when (pred x)
       (cons x (iterate-while pred f (f x))))))
#+END_SRC
** thi.ng.data.byteutils
*** TODO refactor to be CLJS compatible
*** Namespace declaration
#+BEGIN_SRC clojure :tangle babel/src-cljx/thi/ng/data/byteutils.cljx
  (ns thi.ng.data.byteutils
    #+clj
    (:import
     [java.io OutputStream InputStream]))
#+END_SRC
*** Conversion functions
#+BEGIN_SRC clojure :tangle babel/src-cljx/thi/ng/data/byteutils.cljx
  (defn int->byte [x] (if (> x 0x7f) (- x 0x100) x))

  (defn byte->int [x] (if (neg? x) (+ x 0x100) x))
#+END_SRC
*** Write functions
#+BEGIN_SRC clojure :tangle babel/src-cljx/thi/ng/data/byteutils.cljx
  (defn write-str-bytes
    [^OutputStream out ^String x]
    (.write out (.getBytes x))
    out)

  (defn write-int16-le
    [^OutputStream out x]
    (.write
     out
     (byte-array
      [(unchecked-byte (bit-and x 0xff))
       (unchecked-byte (bit-and (bit-shift-right x 8) 0xff))]))
    out)

  (defn write-int32-le
    [^OutputStream out x]
    (.write
     out
     (byte-array
      [(unchecked-byte (bit-and x 0xff))
       (unchecked-byte (bit-and (bit-shift-right x 8) 0xff))
       (unchecked-byte (bit-and (bit-shift-right x 16) 0xff))
       (unchecked-byte (bit-shift-right x 24))]))
    out)

  (defn write-float-le
    [^OutputStream out x]
    (write-int32-le out (Float/floatToRawIntBits x)))

  (defn write-vec3-le
    [^OutputStream out [x y z]]
    (write-float-le out x)
    (write-float-le out y)
    (write-float-le out z))
#+END_SRC
*** Read functions
#+BEGIN_SRC clojure :tangle babel/src-cljx/thi/ng/data/byteutils.cljx
  (defn read-int16-le
    [^InputStream in]
    (let [buf (byte-array 2)]
      (.read in buf 0 2)
      (bit-or (byte->int (aget buf 0)) (bit-shift-left (byte->int (aget buf 1)) 8))))

  (defn read-int32-le
    [^InputStream in]
    (let [buf (byte-array 4)]
      (.read in buf 0 4)
      (bit-or
       (byte->int (aget buf 0))
       (bit-shift-left (byte->int (aget buf 1)) 8)
       (bit-shift-left (byte->int (aget buf 2)) 16)
       (bit-shift-left (byte->int (aget buf 3)) 24))))

  (defn read-float-le
    [^InputStream in]
    (Float/intBitsToFloat (unchecked-int (read-int32-le in))))

  (defn read-vec3-le
    [^InputStream in]
    [(read-float-le in) (read-float-le in) (read-float-le in)])
#+END_SRC
{'fraction_non_alphanumeric': 0.168337211383569, 'fraction_numerical': 0.018972614999105065, 'mean_word_length': 2.334825425246195, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1618863', 'n_tokens_mistral': 4628, 'n_tokens_neox': 4288, 'n_words': 1368}