<commit_before>(ns boot.core
  "The boot core API."
  (:require
    [clojure.java.io             :as io]
    [clojure.set                 :as set]
    [clojure.walk                :as walk]
    [clojure.repl                :as repl]
    [clojure.string              :as string]
    [boot.pod                    :as pod]
    [boot.git                    :as git]
    [boot.cli                    :as cli2]
    [boot.file                   :as file]
    [boot.tmpregistry            :as tmp]
    [boot.tmpdir                 :as tmpd]
    [boot.util                   :as util]
    [boot.from.clojure.tools.cli :as cli])
  (:import
   [java.net URLClassLoader URL]
   java.lang.management.ManagementFactory
   [java.util.concurrent LinkedBlockingQueue TimeUnit]))

(declare watch-dirs sync! on-env! get-env set-env! tmpfile tmpdir ls)

(declare ^{:dynamic true :doc "The running version of boot app."}      *app-version*)
(declare ^{:dynamic true :doc "The running version of boot core."}     *boot-version*)
(declare ^{:dynamic true :doc "Command line options for boot itself."} *boot-opts*)
(declare ^{:dynamic true :doc "Count of warnings during build."}       *warnings*)

;; Internal helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(def ^:private tmpregistry       (atom nil))
(def ^:private cleanup-fns       (atom []))
(def ^:private boot-env          (atom nil))
(def ^:private tempdirs          (atom #{}))
(def ^:private src-watcher       (atom (constantly nil)))

(def ^:private masks
  {:user     {:user true}
   :input    {:input true}
   :output   {:output true}
   :cache    {:input nil :output nil}
   :asset    {:input nil :output true}
   :source   {:input true :output nil}
   :resource {:input true :output true}})

(defn- get-dirs [this masks+]
  (let [dirs        (:dirs this)
        has-mask?   #(= %1 (select-keys %2 (keys %1)))
        filter-keys #(->> %1 (filter (partial has-mask? %2)))]
    (->> masks+ (map masks) (apply merge) (filter-keys dirs) (map tmpfile) set)))

(defn- get-add-dir [this masks+]
  (let [user?  (contains? masks+ :user)
        u-dirs (when-not user? (get-dirs this #{:user}))]
    (-> this (get-dirs masks+) (set/difference u-dirs) first)))

(defn- get-files [this masks+]
  (let [dirs (get-dirs this masks+)]
    (->> this ls (filter (comp dirs tmpdir)) set)))

(defn- temp-dir*
  [key]
  (tmp/mkdir! @tmpregistry key))

(def ^:private new-fileset
  (memoize
    (fn []
      (boot.tmpdir.TmpFileSet. @tempdirs {} (temp-dir* ::blob)))))

(defn- temp-dir**
  [key & masks+]
  (let [k (or key (keyword "boot.core" (str (gensym))))
        m (->> masks+ (map masks) (apply merge))]
    (util/with-let [d (temp-dir* k)]
      (let [t (tmpd/map->TmpDir (assoc m :dir d))]
        (swap! tempdirs conj t)
        (when (:input t)
          (set-env! :directories #(conj % (.getPath (:dir t)))))))))

(defn- add-user-asset    [fileset dir] (tmpd/add fileset (get-add-dir fileset #{:user :asset}) dir))
(defn- add-user-source   [fileset dir] (tmpd/add fileset (get-add-dir fileset #{:user :source}) dir))
(defn- add-user-resource [fileset dir] (tmpd/add fileset (get-add-dir fileset #{:user :resource}) dir))

(defn- user-temp-dirs     [] (get-dirs {:dirs @tempdirs} #{:user}))
(defn- user-asset-dirs    [] (get-dirs {:dirs @tempdirs} #{:user :asset}))
(defn- user-source-dirs   [] (get-dirs {:dirs @tempdirs} #{:user :source}))
(defn- user-resource-dirs [] (get-dirs {:dirs @tempdirs} #{:user :resource}))
(defn- non-user-temp-dirs [] (-> (apply set/union (map :dir @tempdirs))
                                 (set/difference (user-temp-dirs))))

(defn- sync-user-dirs!
  []
  (doseq [[k d] {:source-paths   (user-source-dirs)
                 :resource-paths (user-resource-dirs)
                 :asset-paths    (user-asset-dirs)}]
    (when-let [s (seq (get-env k))]
      (binding [file/*hard-link* false]
        (apply file/sync :theirs (first d) s)))))

(defn- set-user-dirs!
  []
  (@src-watcher)
  (->> [:source-paths :resource-paths :asset-paths]
    (map get-env)
    (apply set/union)
    (watch-dirs (fn [_] (sync-user-dirs!)))
    (reset! src-watcher)))

(defn- do-cleanup!
  []
  (doseq [f @cleanup-fns] (util/guard (f)))
  (reset! cleanup-fns []))

(defn- printable-readable?
  "FIXME: document"
  [form]
  (or (nil? form)<commit_msg>sync directories before returning from set-env!<commit_after>(ns boot.core
  "The boot core API."
  (:require
    [clojure.java.io             :as io]
    [clojure.set                 :as set]
    [clojure.walk                :as walk]
    [clojure.repl                :as repl]
    [clojure.string              :as string]
    [boot.pod                    :as pod]
    [boot.git                    :as git]
    [boot.cli                    :as cli2]
    [boot.file                   :as file]
    [boot.tmpregistry            :as tmp]
    [boot.tmpdir                 :as tmpd]
    [boot.util                   :as util]
    [boot.from.clojure.tools.cli :as cli])
  (:import
   [java.net URLClassLoader URL]
   java.lang.management.ManagementFactory
   [java.util.concurrent LinkedBlockingQueue TimeUnit]))

(declare watch-dirs sync! on-env! get-env set-env! tmpfile tmpdir ls)

(declare ^{:dynamic true :doc "The running version of boot app."}      *app-version*)
(declare ^{:dynamic true :doc "The running version of boot core."}     *boot-version*)
(declare ^{:dynamic true :doc "Command line options for boot itself."} *boot-opts*)
(declare ^{:dynamic true :doc "Count of warnings during build."}       *warnings*)

;; Internal helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(def ^:private tmpregistry       (atom nil))
(def ^:private cleanup-fns       (atom []))
(def ^:private boot-env          (atom nil))
(def ^:private tempdirs          (atom #{}))
(def ^:private src-watcher       (atom (constantly nil)))

(def ^:private masks
  {:user     {:user true}
   :input    {:input true}
   :output   {:output true}
   :cache    {:input nil :output nil}
   :asset    {:input nil :output true}
   :source   {:input true :output nil}
   :resource {:input true :output true}})

(defn- get-dirs [this masks+]
  (let [dirs        (:dirs this)
        has-mask?   #(= %1 (select-keys %2 (keys %1)))
        filter-keys #(->> %1 (filter (partial has-mask? %2)))]
    (->> masks+ (map masks) (apply merge) (filter-keys dirs) (map tmpfile) set)))

(defn- get-add-dir [this masks+]
  (let [user?  (contains? masks+ :user)
        u-dirs (when-not user? (get-dirs this #{:user}))]
    (-> this (get-dirs masks+) (set/difference u-dirs) first)))

(defn- get-files [this masks+]
  (let [dirs (get-dirs this masks+)]
    (->> this ls (filter (comp dirs tmpdir)) set)))

(defn- temp-dir*
  [key]
  (tmp/mkdir! @tmpregistry key))

(def ^:private new-fileset
  (memoize
    (fn []
      (boot.tmpdir.TmpFileSet. @tempdirs {} (temp-dir* ::blob)))))

(defn- temp-dir**
  [key & masks+]
  (let [k (or key (keyword "boot.core" (str (gensym))))
        m (->> masks+ (map masks) (apply merge))]
    (util/with-let [d (temp-dir* k)]
      (let [t (tmpd/map->TmpDir (assoc m :dir d))]
        (swap! tempdirs conj t)
        (when (:input t)
          (set-env! :directories #(conj % (.getPath (:dir t)))))))))

(defn- add-user-asset    [fileset dir] (tmpd/add fileset (get-add-dir fileset #{:user :asset}) dir))
(defn- add-user-source   [fileset dir] (tmpd/add fileset (get-add-dir fileset #{:user :source}) dir))
(defn- add-user-resource [fileset dir] (tmpd/add fileset (get-add-dir fileset #{:user :resource}) dir))

(defn- user-temp-dirs     [] (get-dirs {:dirs @tempdirs} #{:user}))
(defn- user-asset-dirs    [] (get-dirs {:dirs @tempdirs} #{:user :asset}))
(defn- user-source-dirs   [] (get-dirs {:dirs @tempdirs} #{:user :source}))
(defn- user-resource-dirs [] (get-dirs {:dirs @tempdirs} #{:user :resource}))
(defn- non-user-temp-dirs [] (-> (apply set/union (map :dir @tempdirs))
                                 (set/difference (user-temp-dirs))))

(defn- sync-user-dirs!
  []
  (doseq [[k d] {:source-paths   (user-source-dirs)
                 :resource-paths (user-resource-dirs)
                 :asset-paths    (user-asset-dirs)}]
    (when-let [s (seq (get-env k))]
      (binding [file/*hard-link* false]
        (apply file/sync :theirs (first d) s)))))

(defn- set-user-dirs!
  []
  (@src-watcher)
  (->> [:source-paths :resource-paths :asset-paths]
    (map get-env)
    (apply set/union)
    (watch-dirs (fn [_] (sync-user-dirs!)))
    (reset! src-watcher))
  (sync-user-dirs!))

(defn- do-cleanup!
  []
  (doseq [f @cleanup-fns] (util/guard (f)))
  (reset! cleanup-fns []))

(defn- printable-readable?
  "FIXME: document"
  [form]
  (or (nil? form)
{'fraction_non_alphanumeric': 0.18994477680625863, 'fraction_numerical': 0.0013805798435342844, 'mean_word_length': 2.5394951140065145, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2944636', 'n_tokens_mistral': 3231, 'n_tokens_neox': 3132, 'n_words': 833}