<commit_before>(ns spq.lib
  (:require [cheshire.core :as json]
            [clojure.stacktrace :as st]
            [clojure.java.shell :as sh]
            [confs.core :as confs :refer [conf]]
            [durable-queue :as dq]
            [taoensso.timbre :as timbre]))

(def *kill* (doto (not= \n (first (System/getenv "KILL")))
              (->> (println "kill:"))))

(defn run [& args]
  (let [cmd (apply str (interpose " " args))
        res (clojure.java.shell/sh "bash" "-c" cmd)]
    (assert (-> res :exit (= 0)) (assoc res :cmd cmd))
    (.trim (:out res))))

(defn setup-logging
  [& [short-format]]
  (timbre/merge-config!
   {:appenders
    {:println
     {:fn (let [atomic-print (fn [x]
                               (print x)
                               (flush))]
            (if short-format
              #(let [{:keys [msg_ ?err_]} %]
                 (atomic-print
                  (str
                   (force msg_)
                   " "
                   (if-let [err (force ?err_)]
                     (str "\n" (st/print-stack-trace err))
                     "")
                   "\n")))
              #(let [{:keys [msg_ hostname_ timestamp_ instant level ?ns-str ?err_]} %]
                 (atomic-print
                  (str
                   (.toString (.toInstant instant))
                   " "
                   level
                   " "
                   (force ?ns-str)
                   " "
                   (force hostname_)
                   " "
                   (force msg_)
                   " "
                   (if-let [err (force ?err_)]
                     (str "\n" (st/print-stack-trace err))
                     "")
                   "\n")))))}}}))

(defn shutdown
  []
  (when *kill*
    (timbre/fatal "shutdown. if jvm doesnt exit right after this, it should have")
    (Thread/sleep 1000)
    (shutdown-agents)
    (System/exit 0) ;; soft kill
    (run "ps h -o ppid $$|xargs sudo kill -9") ;;  hard kill
    nil))<commit_msg>adjust logging level<commit_after>(ns spq.lib
  (:require [cheshire.core :as json]
            [clojure.stacktrace :as st]
            [clojure.java.shell :as sh]
            [confs.core :as confs :refer [conf]]
            [durable-queue :as dq]
            [taoensso.timbre :as timbre]))

(def *kill* (doto (not= \n (first (System/getenv "KILL")))
              (->> (println "kill:"))))

(defn run [& args]
  (let [cmd (apply str (interpose " " args))
        res (clojure.java.shell/sh "bash" "-c" cmd)]
    (assert (-> res :exit (= 0)) (assoc res :cmd cmd))
    (.trim (:out res))))

(defn setup-logging
  [& [short-format]]
  (timbre/merge-config!
   {:appenders
    {:println
     {:fn (let [atomic-print (fn [x]
                               (print x)
                               (flush))]
            (if short-format
              #(let [{:keys [msg_ ?err_]} %]
                 (atomic-print
                  (str
                   (force msg_)
                   " "
                   (if-let [err (force ?err_)]
                     (str "\n" (st/print-stack-trace err))
                     "")
                   "\n")))
              #(let [{:keys [msg_ hostname_ timestamp_ instant level ?ns-str ?err_]} %]
                 (atomic-print
                  (str
                   (.toString (.toInstant instant))
                   " "
                   level
                   " "
                   (force ?ns-str)
                   " "
                   (force hostname_)
                   " "
                   (force msg_)
                   " "
                   (if-let [err (force ?err_)]
                     (str "\n" (st/print-stack-trace err))
                     "")
                   "\n")))))}}}))

(defn shutdown
  []
  (when *kill*
    (timbre/warn "shutdown. if jvm doesnt exit right after this, it should have")
    (Thread/sleep 1000)
    (shutdown-agents)
    (System/exit 0) ;; soft kill
    (run "ps h -o ppid $$|xargs sudo kill -9") ;;  hard kill
    nil))
{'fraction_non_alphanumeric': 0.14626865671641792, 'fraction_numerical': 0.003482587064676617, 'mean_word_length': 1.153722549544724, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7508909', 'n_tokens_mistral': 1268, 'n_tokens_neox': 1188, 'n_words': 313}