<commit_before>(ns repodoc.utils)

(defn dsu-sort
  "Decorate - sort - undecorate
  from http://gettingclojure.wikidot.com/cookbook:sequences
  Also used in Python Cookbook
  This version sorts only on decorated values directly"
  [sortable decorator]
  (map last
     (sort (fn [[v1 a] [v2 b]]
             (< v1 v2))
           (map (fn [x]
                  [(decorator x) x])
                sortable))))

(defn sapling
  "Function that reduces into acc
  a map of given items' basenames
  using directory paths as keys"
  [acc item]
  (let [path (get item "path")
        parts (clojure.string/split path #"/")
        dirs (clojure.string/join "/" (butlast parts))
        dir (if (empty? dirs) "/" dirs)
        leaf (get acc dir)]
    (if leaf
      (assoc acc dir (conj leaf item))
      (assoc acc dir [item]))))

(defn path-reduce [paths]
  (reduce sapling  {} paths))

(defn vector-compare [[value1 & rest1] [value2 & rest2]]
  "From http://stackoverflow.com/questions/15716894/"
  (let [result (compare value1 value2)]
    (cond
      (not (= result 0)) result
      (nil? value1) 0 ; value2 will be nil as well
      :else (recur rest1 rest2))))
<commit_msg>Added utility functions<commit_after>(ns repodoc.utils)

(defn dsu-sort
  "Decorate - sort - undecorate
  from http://gettingclojure.wikidot.com/cookbook:sequences
  Also used in Python Cookbook
  This version sorts only on decorated values directly"
  [sortable decorator]
  (map last
     (sort (fn [[v1 a] [v2 b]]
             (< v1 v2))
           (map (fn [x]
                  [(decorator x) x])
                sortable))))

(defn sapling
  "Function that reduces into acc
  a map of given items' basenames
  using directory paths as keys"
  [acc item]
  (let [path (get item "path")
        parts (clojure.string/split path #"/")
        dirs (clojure.string/join "/" (butlast parts))
        dir (if (empty? dirs) "/" dirs)
        leaf (get acc dir)]
    (if leaf
      (assoc acc dir (conj leaf item))
      (assoc acc dir [item]))))

(defn path-reduce [paths]
  (reduce sapling  {} paths))

(defn vector-compare [[value1 & rest1] [value2 & rest2]]
  "From http://stackoverflow.com/questions/15716894/"
  (let [result (compare value1 value2)]
    (cond
      (not (= result 0)) result
      (nil? value1) 0 ; value2 will be nil as well
      :else (recur rest1 rest2))))

(defn add-to-map
  "Addind values into :children of keys path
  trying to keep whatever other values
  the target map has"
  [data keys val]
  (update-in data
             keys
             (fn [old]
               (if (nil? old)
                 {:children [val]}
                 (if (:children old)
                   (update old :children
                           (fn [older & arg]
                             (conj older val))
                   (assoc old :children [val])))))))

;; Binary tree from JoC 2nd edition listing 9.1
(defrecord TreeNode [val l r])

(defn xconj [t v]
  (println t v)
  (cond
    (nil? t)       (TreeNode. v nil nil)
    (< v (:val t)) (TreeNode. (:val t) (xconj (:l t) v) (:r t))
    :else          (TreeNode. (:val t) (:l t) (xconj (:r t) v))))

(defn xseq [t]
  (when t
    (concat (xseq (:l t)) [(:val t)] (xseq (:r t)))))

(def sample-tree (reduce xconj nil [3 5 2 4 6]))
(xseq sample-tree)

(defrecord GitTree [name tree])

(defn build-git-tree [t name tree]
  "Build a tree node for given name
  and add given list of names from the tree under it"
  (cond
    (nil? tree) (GitTree. name nil "blob")
    :else (GitTree. name (map build-git-tree tree) "tree")))

(defn min-vec
  "Return largest vectors length"
  [vecs]
  (apply min (map count vecs)))

(defn cut-vec
  "Returns vectors cut to the size
  of smallest vector"
  [& vecs]
  (let [size (min-vec vecs)]
    (map #(subvec % 0 (min (count %) size)) vecs)))

(defn sort-paths
  "Sort paths lexicographically by comparing
  path items directly and cutting vec length to
  equal"
  [paths]
  (sort #(apply compare (cut-vec %1 %2)) paths))

{'fraction_non_alphanumeric': 0.12957393483709273, 'fraction_numerical': 0.014786967418546366, 'mean_word_length': 2.370777027027027, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1654946', 'n_tokens_mistral': 1400, 'n_tokens_neox': 1339, 'n_words': 487}