<commit_before>(ns funcatron.tron.util
  "Utilities for Tron"
  (:require [cheshire.core :as json]
            [org.httpkit.server :as kit]
            [cognitect.transit :as transit]
            [io.sarnowski.swagger1st.context :as s1ctx]
            [camel-snake-kebab.core :as csk]
            [org.httpkit.client :as http]
            [taoensso.timbre :as timbre
             :refer [log trace debug info warn error fatal report
                     logf tracef debugf infof warnf errorf fatalf reportf
                     spy get-env]]
            [clojure.java.shell :as shelly]
            [clojure.spec :as s]
            [clojure.spec.test :as stest])
  (:import (cheshire.prettyprint CustomPrettyPrinter)
           (java.util Base64 Map Map$Entry List UUID Properties)
           (org.apache.commons.io IOUtils)
           (java.io InputStream ByteArrayInputStream ByteArrayOutputStream File FileInputStream OutputStreamWriter StringReader)
           (com.fasterxml.jackson.databind ObjectMapper)
           (javax.xml.parsers DocumentBuilderFactory)
           (org.w3c.dom Node)
           (javax.xml.transform TransformerFactory OutputKeys)
           (javax.xml.transform.dom DOMSource)
           (javax.xml.transform.stream StreamResult)
           (clojure.lang IFn)
           (funcatron.abstractions Router$Message)
           (java.security MessageDigest)
           (java.util.jar JarFile JarEntry)
           (java.util.concurrent Executors ScheduledExecutorService TimeUnit)
           (java.util.function Function BiFunction)
           (funcatron.helpers Tuple2 Tuple3 LoggerBridge LoggerBridge$ActualLogger)
           (com.spotify.dns DnsSrvResolvers DnsSrvResolver LookupResult)
           (java.lang.management ManagementFactory ThreadMXBean RuntimeMXBean)
           (com.fasterxml.jackson.module.paramnames ParameterNamesModule)
           (com.fasterxml.jackson.datatype.jdk8 Jdk8Module)
           (com.fasterxml.jackson.datatype.jsr310 JavaTimeModule)
           (java.util.logging Logger LogRecord Level)
           (com.fasterxml.jackson.core JsonParser$Feature)))


(set! *warn-on-reflection* true)

(def ^ObjectMapper jackson-json
  "The Jackson JSON ObjectMapper"
  (let [jackson (->
                  (.findAndRegisterModules (ObjectMapper.))
                  (.registerModule (ParameterNamesModule.))
                  (.registerModule (Jdk8Module.))
                  (.registerModule (JavaTimeModule.))
                  )]
    (.configure jackson JsonParser$Feature/ALLOW_SINGLE_QUOTES true)
    jackson
    ))

(defn walk
  "Walk Clojure data structures and 'do the right thing'"
  [element-f key-f data]
  (let [m (element-f data)
        f (partial walk element-f key-f)]
    (cond

      (list? m) (map f m)
      (instance? List m) (mapv f m)
      (instance? Map m) (into {} (map (fn [^Map$Entry me]
                                        (let [k (.getKey me)]
                                          [(key-f k) (f (.getValue me))]
                                          )) (.entrySet ^Map m)))

      :else m
      )
    )
  )


(s/fdef kwd-to-string
  :args (s/cat :kw? keyword?)
  :ret string?)
(defn kwd-to-string
  "Converts a keyword to a nsless String"
  [kw?]
  (if (keyword? kw?) (name kw?) kw?))

(stest/instrument 'kwd-to-string)

(s/fdef string-to-kwd
  :args (s/cat :s? string?)
  :ret keyword?)
(defn string-to-kwd
  "Converts a String to a Keyword"
  [s?]
  (if (string? s?) (keyword s?) s?))

(stest/instrument 'string-to-kwd)

(defn stringify-keys
  "Recursively transforms all map keys from keywords to strings."
  ([m] (stringify-keys identity m))
  ([f m] (walk f kwd-to-string m))
  )

(defn camel-stringify-keys
  "Recursively transforms all map keys from keywords to strings."
  ([m] (camel-stringify-keys identity m))
  ([f m] (walk f csk/->camelCaseString m))
  )

(defn keywordize-keys
  "Recursively transforms all map keys from keywords to strings."
  ([m] (keywordize-keys identity m))
  ([f m] (walk f string-to-kwd m))
  )

(defn kebab-keywordize-keys
  "Recursively transforms all map keys from keywords to strings."
  ([m] (kebab-keywordize-keys identity m))<commit_msg>Turned off instrumentation in prod<commit_after>(ns funcatron.tron.util
  "Utilities for Tron"
  (:require [cheshire.core :as json]
            [org.httpkit.server :as kit]
            [cognitect.transit :as transit]
            [io.sarnowski.swagger1st.context :as s1ctx]
            [camel-snake-kebab.core :as csk]
            [org.httpkit.client :as http]
            [taoensso.timbre :as timbre
             :refer [log trace debug info warn error fatal report
                     logf tracef debugf infof warnf errorf fatalf reportf
                     spy get-env]]
            [clojure.java.shell :as shelly]
            [clojure.spec :as s]
            )
  (:import (cheshire.prettyprint CustomPrettyPrinter)
           (java.util Base64 Map Map$Entry List UUID Properties)
           (org.apache.commons.io IOUtils)
           (java.io InputStream ByteArrayInputStream ByteArrayOutputStream File FileInputStream OutputStreamWriter StringReader)
           (com.fasterxml.jackson.databind ObjectMapper)
           (javax.xml.parsers DocumentBuilderFactory)
           (org.w3c.dom Node)
           (javax.xml.transform TransformerFactory OutputKeys)
           (javax.xml.transform.dom DOMSource)
           (javax.xml.transform.stream StreamResult)
           (clojure.lang IFn)
           (funcatron.abstractions Router$Message)
           (java.security MessageDigest)
           (java.util.jar JarFile JarEntry)
           (java.util.concurrent Executors ScheduledExecutorService TimeUnit)
           (java.util.function Function BiFunction)
           (funcatron.helpers Tuple2 Tuple3 LoggerBridge LoggerBridge$ActualLogger)
           (com.spotify.dns DnsSrvResolvers DnsSrvResolver LookupResult)
           (java.lang.management ManagementFactory ThreadMXBean RuntimeMXBean)
           (com.fasterxml.jackson.module.paramnames ParameterNamesModule)
           (com.fasterxml.jackson.datatype.jdk8 Jdk8Module)
           (com.fasterxml.jackson.datatype.jsr310 JavaTimeModule)
           (java.util.logging Logger LogRecord Level)
           (com.fasterxml.jackson.core JsonParser$Feature)))


(set! *warn-on-reflection* true)

(def ^ObjectMapper jackson-json
  "The Jackson JSON ObjectMapper"
  (let [jackson (->
                  (.findAndRegisterModules (ObjectMapper.))
                  (.registerModule (ParameterNamesModule.))
                  (.registerModule (Jdk8Module.))
                  (.registerModule (JavaTimeModule.))
                  )]
    (.configure jackson JsonParser$Feature/ALLOW_SINGLE_QUOTES true)
    jackson
    ))

(defn walk
  "Walk Clojure data structures and 'do the right thing'"
  [element-f key-f data]
  (let [m (element-f data)
        f (partial walk element-f key-f)]
    (cond

      (list? m) (map f m)
      (instance? List m) (mapv f m)
      (instance? Map m) (into {} (map (fn [^Map$Entry me]
                                        (let [k (.getKey me)]
                                          [(key-f k) (f (.getValue me))]
                                          )) (.entrySet ^Map m)))

      :else m
      )
    )
  )


(s/fdef kwd-to-string
  :args (s/cat :kw? keyword?)
  :ret string?)
(defn kwd-to-string
  "Converts a keyword to a nsless String"
  [kw?]
  (if (keyword? kw?) (name kw?) kw?))

(s/fdef string-to-kwd
  :args (s/cat :s? string?)
  :ret keyword?)
(defn string-to-kwd
  "Converts a String to a Keyword"
  [s?]
  (if (string? s?) (keyword s?) s?))

(defn stringify-keys
  "Recursively transforms all map keys from keywords to strings."
  ([m] (stringify-keys identity m))
  ([f m] (walk f kwd-to-string m))
  )

(defn camel-stringify-keys
  "Recursively transforms all map keys from keywords to strings."
  ([m] (camel-stringify-keys identity m))
  ([f m] (walk f csk/->camelCaseString m))
  )

(defn keywordize-keys
  "Recursively transforms all map keys from keywords to strings."
  ([m] (keywordize-keys identity m))
  ([f m] (walk f string-to-kwd m))
  )

(defn kebab-keywordize-keys
  "Recursively transforms all map keys from keywords to strings."
  ([m] (kebab-keywordize-keys identity m))
{'fraction_non_alphanumeric': 0.10681377303588749, 'fraction_numerical': 0.0031522793404461687, 'mean_word_length': 2.300920368147259, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2119736', 'n_tokens_mistral': 2619, 'n_tokens_neox': 2507, 'n_words': 676}