<commit_before>(ns muon-clojure.utils
  (:require [clojure.tools.logging :as log]))

(defn dekeywordize
  "Converts the keys in a map from keywords to strings."
  [m]
  (if (map? m)
    (apply merge
           (map (fn [[k v]] {(if (keyword? k)
                               (name k)
                               (str k))
                             (if (map? v)
                               (dekeywordize v)
                               (if (sequential? v)
                                 (into (empty v) (map dekeywordize v))
                                 (if (keyword? v)
                                   (name v)
                                   v)))})
                m))
    (if (sequential? m)
      (into (empty m) (map dekeywordize m))
      (if (keyword? m)
        (name m)
        m))))

(defn keywordize
  "Converts the keys in a map from strings to keywords"
  [m]
  (log/info "keywordize" (class m) ":" (pr-str m))
  (if (instance? com.google.gson.internal.LinkedTreeMap m)
    (keywordize (into {} m))
    (if (map? m)
      (apply merge
             (map (fn [[k v]] {(if (keyword? k)
                                 k
                                 (keyword k))
                               (if (instance? com.google.gson.internal.LinkedTreeMap v)
                                 (keywordize (into {} v))
                                 (if (map? v)
                                   (keywordize v)
                                   (if (instance? java.util.ArrayList v)
                                     (keywordize (into [] v))
                                     (if (sequential? v)
                                       (into (empty v) (map keywordize v))
                                       v))))})
                  m))
      (if (instance? java.util.ArrayList m)
        (keywordize (into [] m))
        (if (sequential? m)
          (into (empty m) (map keywordize m))
          m)))))

<commit_msg>Always use vectors for seqs when dekeywordising<commit_after>(ns muon-clojure.utils
  (:require [clojure.tools.logging :as log]))

(defn dekeywordize
  "Converts the keys in a map from keywords to strings."
  [m]
  (if (map? m)
    (apply merge
           (map (fn [[k v]] {(if (keyword? k)
                               (name k)
                               (str k))
                             (if (map? v)
                               (dekeywordize v)
                               (if (sequential? v)
                                 (into [] (map dekeywordize v))
                                 (if (keyword? v)
                                   (name v)
                                   v)))})
                m))
    (if (sequential? m)
      (into [] (map dekeywordize m))
      (if (keyword? m)
        (name m)
        m))))

(defn keywordize
  "Converts the keys in a map from strings to keywords"
  [m]
  (log/info "keywordize" (class m) ":" (pr-str m))
  (if (instance? com.google.gson.internal.LinkedTreeMap m)
    (keywordize (into {} m))
    (if (map? m)
      (apply merge
             (map (fn [[k v]] {(if (keyword? k)
                                 k
                                 (keyword k))
                               (if (instance? com.google.gson.internal.LinkedTreeMap v)
                                 (keywordize (into {} v))
                                 (if (map? v)
                                   (keywordize v)
                                   (if (instance? java.util.ArrayList v)
                                     (keywordize (into [] v))
                                     (if (sequential? v)
                                       (into (empty v) (map keywordize v))
                                       v))))})
                  m))
      (if (instance? java.util.ArrayList m)
        (keywordize (into [] m))
        (if (sequential? m)
          (into (empty m) (map keywordize m))
          m)))))


{'fraction_non_alphanumeric': 0.11128048780487805, 'fraction_numerical': 0.0, 'mean_word_length': 0.9139523578026252, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4771923', 'n_tokens_mistral': 1057, 'n_tokens_neox': 1028, 'n_words': 320}