<commit_before>(ns search.wrappers.graph
  (:require [schema.core :as s]
            [plumbing.core :refer [fnk]]
            [taoensso.timbre.profiling :as profiling]

            [search.utils :as utils]
            [search.core :as search]))

(defn println-level
  "Like println but indents by `indent` first."
  [indent & messages]
  (apply
    println
    (concat (repeat (* 2 indent) "")
            messages)))

(s/defn log-wrap :- utils/Graph
  "Modify graph spec g, producing a new graph spec.

  If any node produces a function, it wraps that function to log when
  it is called."
  [g :- utils/Graph]
  (let [indent (atom 0)]
    (utils/map-leaf-fns
      (fnk log-wrap-inner [f ks]
        (-> f
          (utils/wrap-before (fn [& _] (println-level @indent "Calling" ks) (swap! indent inc)))
          (utils/wrap-after (fn [_] (swap! indent dec) (println-level @indent "Finished" ks)))))
      g)))

(def profile-fns-wrap
  "Add timbre profile to each function"
  (partial utils/map-leaf-fns
    (fnk profile-wrap-inner [f ks]
      (utils/wrap f (fn [& args] (profiling/p (str ks) (apply f args)))))))

(s/defn print-profile-gen-wrap :- search/SearchGraph
  "Prints the total profiling after each generation"
  [{->gens :generations :as g} :- search/SearchGraph]
  (assoc g :generations
    (utils/wrap ->gens
      (fn [m]
        (let [f (fn profile-fns-inner [gens]
                  (let [first-g (profiling/profile :info :Generation (first gens))]
                    (when-not (nil? first-g)
                      (lazy-seq (cons first-g
                                      (profile-fns-inner (rest gens)))))))]
          (f (->gens m)))))))
<commit_msg>Add some more profiling that might not be useful<commit_after>(ns search.wrappers.graph
  (:require [schema.core :as s]
            [plumbing.core :refer [fnk]]
            [taoensso.timbre.profiling :as profiling]

            [search.utils :as utils]
            [search.core :as search]))

(defn println-level
  "Like println but indents by `indent` first."
  [indent & messages]
  (apply
    println
    (concat (repeat (* 2 indent) "")
            messages)))

(s/defn log-wrap :- utils/Graph
  "Modify graph spec g, producing a new graph spec.

  If any node produces a function, it wraps that function to log when
  it is called.

  This is useful when debugging where your application is spending time."
  [g :- utils/Graph]
  (let [indent (atom 0)]
    (utils/map-leaf-fns
      (fnk log-wrap-inner [f ks]
        (-> f
          (utils/wrap-before (fn [& _] (println-level @indent "Calling" ks) (swap! indent inc)))
          (utils/wrap-after (fn [_] (swap! indent dec) (println-level @indent "Finished" ks)))))
      g)))


(s/defn profile-wrap :- utils/Graph
  "Print timbre profiling after each graph key is generated"
  [g :- utils/Graph]
  (plumbing.map/map-leaves-and-path
    (fn [ks f]
      (utils/wrap
        f
        (fn [& args]
          (profiling/profile :info :Graph
            (profiling/p (str ks) (apply f args))))))
    g))

(defn wrap-fn-profile
  [label f]
  (utils/wrap f (fn [& args] (profiling/p label (apply f args)))))

(def profile-fns-wrap
  "Add timbre profile to each function"
  (partial utils/map-leaf-fns
    (fnk profile-wrap-inner [ks f]
      (wrap-fn-profile (str ks :inner) f))))

(s/defn print-profile-gen-wrap :- search/SearchGraph
  "Prints the total profiling after each generation"
  [{->gens :generations :as g} :- search/SearchGraph]
  (assoc g :generations
    (utils/wrap ->gens
      (fn [m]
        (let [f (fn profile-fns-inner [gens]
                  (let [first-g (profiling/profile :info :Generation (first gens))]
                    (when-not (nil? first-g)
                      (lazy-seq (cons first-g
                                      (profile-fns-inner (rest gens)))))))]
          (f (->gens m)))))))

{'fraction_non_alphanumeric': 0.14022809745982373, 'fraction_numerical': 0.0010368066355624676, 'mean_word_length': 2.242857142857143, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3602170', 'n_tokens_mistral': 1299, 'n_tokens_neox': 1261, 'n_words': 389}