<commit_before>(ns asset-minifier.core
  (:require [clojure.java.io :refer [file reader writer make-parents]]
            [clj-html-compressor.core :as html-comressor])
  (:import com.yahoo.platform.yui.compressor.CssCompressor
           java.util.zip.GZIPOutputStream
           java.io.FileInputStream
           java.io.FileOutputStream
           java.io.File
           org.apache.commons.io.IOUtils
           java.util.logging.Level
           java.nio.charset.Charset
           clojure.lang.Sequential
           [com.google.javascript.jscomp
            CompilationLevel
            CompilerOptions
            SourceFile
            CompilerOptions$LanguageMode]))

(defn- delete-target [target]
  (let [f (file target)]
    (when (.exists f)
      (.delete f))
    (make-parents f)))

(defn- find-assets [f ext]
  (if (.isDirectory f)
    (->> f
         file-seq
         (filter (fn [file] (-> file .getName (.endsWith ext)))))
    [f]))

(defn- aggregate [path ext]
  (if (coll? path)
    (flatten
     (for [item path]
       (let [f (file item)]
         (find-assets f ext))))
    (let [f (file path)]
      (find-assets f ext))))

(defn total-size [files]
  (->> files (map #(.length %)) (apply +)))

(defn- create-temp-file [source]
  (File/createTempFile (.getName source) ".gz"))

(defn- create-temp-files [sources]
  (map create-temp-file sources))

(defn- gzip [file-in file-out]
  (with-open [in (FileInputStream. file-in)
              out (FileOutputStream. file-out)
              outGZIP (GZIPOutputStream. out)]
    (IOUtils/copy in outGZIP)))

(defmulti compression-details (fn [& args] (mapv class args)))
(defmethod compression-details [Sequential File] [sources target]
  (let [tmp (create-temp-file target)]
    (gzip target tmp)
    (let [uncompressed-length (total-size sources)
          compressed-length   (.length target)]
      {:sources (map #(.getName %) sources)
       :target (.getName target)
       :original-size uncompressed-length
       :compressed-size compressed-length
       :gzipped-size (.length tmp)})))
(defmethod compression-details [Sequential Sequential] [sources targets]
  (let [tmps (create-temp-files sources)]
    (doseq [source sources
            tmp tmps]
      (gzip source tmp))
    (let [uncompressed-length (total-size sources)
          compressed-length (total-size targets)
          gziped-length (total-size tmps)]
      {:sources (map #(.getName %) sources)
       :targets (map #(.getName %) targets) ;; Here i use targets
       :original-size uncompressed-length
       :compressed-size compressed-length
       :gzipped-size gziped-length})))

(defn minify-css-input [source target {:keys [linebreak] :or {linebreak -1}}]
  (with-open [rdr (reader source)
              wrt (writer target)]
    (-> rdr (CssCompressor.) (.compress wrt linebreak))))

(defn minify-css [path target & [opts]]
  (delete-target target)
  (let [assets (aggregate path ".css")
        tmp    (File/createTempFile "temp-sources" ".css")
        target (file target)]
    (with-open [wrt (writer tmp :append true)]
      (doseq [file assets]
        (.append wrt (slurp file))))
    (minify-css-input tmp target opts)
    (compression-details assets target)))

(defn- set-optimization [options optimization]
  (-> optimization
      {:advanced CompilationLevel/ADVANCED_OPTIMIZATIONS
       :simple CompilationLevel/SIMPLE_OPTIMIZATIONS
       :whitespace CompilationLevel/WHITESPACE_ONLY}
      (.setOptionsForCompilationLevel options))
  options)

(defn- parse-language [language]
  (-> language
      (name)
      (.toUpperCase)
      (CompilerOptions$LanguageMode/fromString)))

(defn- compile-js [compiler assets externs optimization language]
  (let [language (parse-language language)
        options  (-> (doto (CompilerOptions.)
                       (.setLanguageIn language)
                       (.setLanguageOut language)
                       (.setOutputCharset (Charset/forName "UTF-8")))
                     (set-optimization optimization))]

    (.compile compiler
              (map #(SourceFile/fromFile %) externs)
              (map #(SourceFile/fromFile %) assets)
              options)
    {:warnings (map #(.toString %) (.getWarnings compiler))
     :errors   (map #(.toString %) (.getErrors compiler))}))

(defn merge-files [sources target]
  (with-open [out (FileOutputStream. target)]
    (doseq [file sources]
      (with-open [in (FileInputStream. file)]
        (IOUtils/copy in out))))
  {:sources (map #(.getName %) sources)
   :target (.getName (file target))
   :original-size (total-size sources)})

(defn minify-js [path target & [{:keys [quiet? externs optimization language]
                                 :or {quiet? false
                                      language :ecmascript5
                                      externs []
                                      optimization :simple}}]]
  (delete-target target)
  (if (= :none optimization)
    (merge-files (aggregate path ".js") target)
    (do
      (if quiet?
        (com.google.javascript.jscomp.Compiler/setLoggingLevel Level/OFF)
        (com.google.javascript.jscomp.Compiler/setLoggingLevel Level/SEVERE))
      (let [assets   (aggregate path ".js")
            compiler (com.google.javascript.jscomp.Compiler.)
            result   (compile-js compiler assets externs optimization language)]
        (spit target (.toSource compiler))
        (merge result (compression-details assets (file target)))))))

(defn minify-html-asset [asset target opts]
  (let [result (html-comressor/compress (slurp asset))
        target-filename (str target (.getName asset))]
    (make-parents target-filename)
    (spit target-filename result)))

(defn minify-html [path target opts]
  (delete-target target)
  (let [assets (aggregate path ".html")]
    (doseq [asset assets]
      (minify-html-asset asset target opts))
    (compression-details assets (aggregate target ".html"))))

(defn minify
  "assets are specified using a map where the key is the output file and the value is the asset paths, eg:
  {\"site.min.css\" \"dev/resources/css\"
  \"site.min.js\" \"dev/resources/js/site.js\"
  \"vendor.min.js\" [\"dev/resources/vendor1/js\"
  \"dev/resources/vendor2/js\"]}"
  [assets & [opts]]
  (into {}
        (for [[target path] assets]
          [[path target]
           (cond
             (.endsWith target ".js")  (minify-js path target opts)
             (.endsWith target ".css") (minify-css path target opts)
             :else (throw (ex-info "unrecognized target" target)))])))<commit_msg>refactor(core): make code more readable<commit_after>(ns asset-minifier.core
  (:require [clojure.java.io :refer [file reader writer make-parents]]
            [clojure.string :as string]
            [clj-html-compressor.core :as html-comressor])
  (:import com.yahoo.platform.yui.compressor.CssCompressor
           java.util.zip.GZIPOutputStream
           java.io.FileInputStream
           java.io.FileOutputStream
           java.io.File
           org.apache.commons.io.IOUtils
           java.util.logging.Level
           java.nio.charset.Charset
           clojure.lang.Sequential
           [com.google.javascript.jscomp
            CompilationLevel
            CompilerOptions
            SourceFile
            CompilerOptions$LanguageMode]))

(def js ".js")
(def css ".css")
(def html ".html")
(def gz ".gz")

(defn- delete-target [target]
  (let [f (file target)]
    (when (.exists f)
      (.delete f))
    (make-parents f)))

(defn- find-assets [folder ext]
  (if (not (.isDirectory folder))
    [folder]
    (->> folder
         (file-seq)
         (filter #(string/ends-with? (.getName %) ext)))))

(defn- aggregate [paths ext]
  (if (not (coll? paths))
    (aggregate [paths] ext)
    (->> paths
         (map #(find-assets (file %) ext))
         (flatten))))

(defn total-size [files]
  (->> files 
       (map #(.length %))
       (apply +)))

(defn- create-temp-file [source]
  (File/createTempFile (.getName source) gz))

(defn- create-temp-files [sources]
  (map create-temp-file sources))

(defn- gzip [file-in file-out]
  (with-open [in (FileInputStream. file-in)
              out (FileOutputStream. file-out)
              outGZIP (GZIPOutputStream. out)]
    (IOUtils/copy in outGZIP)))

(defmulti compression-details (fn [& args] (mapv class args)))
(defmethod compression-details [Sequential File] [sources target]
  (let [tmp (create-temp-file target)]
    (gzip target tmp)
    (let [uncompressed-length (total-size sources)
          compressed-length   (.length target)]
      {:sources (map #(.getName %) sources)
       :target (.getName target)
       :original-size uncompressed-length
       :compressed-size compressed-length
       :gzipped-size (.length tmp)})))
(defmethod compression-details [Sequential Sequential] [sources targets]
  (let [tmps (create-temp-files sources)]
    (doseq [source sources
            tmp tmps]
      (gzip source tmp))
    (let [uncompressed-length (total-size sources)
          compressed-length (total-size targets)
          gziped-length (total-size tmps)]
      {:sources (map #(.getName %) sources)
       :targets (map #(.getName %) targets) ;; Here i use targets
       :original-size uncompressed-length
       :compressed-size compressed-length
       :gzipped-size gziped-length})))

(defn minify-css-input [source target {:keys [linebreak] :or {linebreak -1}}]
  (with-open [rdr (reader source)
              wrt (writer target)]
    (-> rdr 
        (CssCompressor.) 
        (.compress wrt linebreak))))

(defn minify-css [path target & [opts]]
  (delete-target target)
  (let [assets (aggregate path css)
        tmp    (File/createTempFile "temp-sources" css)
        target (file target)]
    (with-open [wrt (writer tmp :append true)]
      (doseq [file assets]
        (.append wrt (slurp file))))
    (minify-css-input tmp target opts)
    (compression-details assets target)))

(defn- set-optimization [options optimization]
  (-> optimization
      {:advanced CompilationLevel/ADVANCED_OPTIMIZATIONS
       :simple CompilationLevel/SIMPLE_OPTIMIZATIONS
       :whitespace CompilationLevel/WHITESPACE_ONLY}
      (.setOptionsForCompilationLevel options))
  options)

(defn- parse-language [language]
  (-> language
      (name)
      (.toUpperCase)
      (CompilerOptions$LanguageMode/fromString)))

(defn- compile-js [compiler assets externs optimization language]
  (let [language (parse-language language)
        options  (-> (doto (CompilerOptions.)
                       (.setLanguageIn language)
                       (.setLanguageOut language)
                       (.setOutputCharset (Charset/forName "UTF-8")))
                     (set-optimization optimization))]

    (.compile compiler
              (map #(SourceFile/fromFile %) externs)
              (map #(SourceFile/fromFile %) assets)
              options)
    {:warnings (map #(.toString %) (.getWarnings compiler))
     :errors   (map #(.toString %) (.getErrors compiler))}))

(defn merge-files [sources target]
  (with-open [out (FileOutputStream. target)]
    (doseq [file sources]
      (with-open [in (FileInputStream. file)]
        (IOUtils/copy in out))))
  {:sources (map #(.getName %) sources)
   :target (.getName (file target))
   :original-size (total-size sources)})

(defn minify-js [path target & [{:keys [quiet? externs optimization language]
                                 :or {quiet? false
                                      language :ecmascript5
                                      externs []
                                      optimization :simple}}]]
  (delete-target target)
  (if (= :none optimization)
    (merge-files (aggregate path js) target)
    (do
      (if quiet?
        (com.google.javascript.jscomp.Compiler/setLoggingLevel Level/OFF)
        (com.google.javascript.jscomp.Compiler/setLoggingLevel Level/SEVERE))
      (let [assets   (aggregate path js)
            compiler (com.google.javascript.jscomp.Compiler.)
            result   (compile-js compiler assets externs optimization language)]
        (spit target (.toSource compiler))
        (merge result (compression-details assets (file target)))))))

(defn minify-html-asset [asset target opts]
  (let [result (html-comressor/compress (slurp asset))
        target-filename (str target (.getName asset))]
    (make-parents target-filename)
    (spit target-filename result)))

(defn minify-html [path target opts]
  (delete-target target)
  (let [assets (aggregate path html)]
    (doseq [asset assets]
      (minify-html-asset asset target opts))
    (compression-details assets (aggregate target html))))

(defn minify
  "assets are specified using a map where the key is the output file and the value is the asset paths, eg:
  {\"site.min.css\" \"dev/resources/css\"
  \"site.min.js\" \"dev/resources/js/site.js\"
  \"vendor.min.js\" [\"dev/resources/vendor1/js\"
  \"dev/resources/vendor2/js\"]}"
  [assets & [opts]]
  (into {}
        (for [[target path] assets]
          [[path target]
           (cond
             (.endsWith target js)  (minify-js path target opts)
             (.endsWith target css) (minify-css path target opts)
             :else (throw (ex-info "unrecognized target" target)))])))
{'fraction_non_alphanumeric': 0.13092993438419187, 'fraction_numerical': 0.0007542046911531789, 'mean_word_length': 2.466666666666667, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1654918', 'n_tokens_mistral': 4066, 'n_tokens_neox': 3906, 'n_words': 1050}