<commit_before>(ns migae.boot-gae
  {:boot/export-tasks true}
  (:require [clojure.java.io :as io]
            [clojure.string :as str]
            [clojure.set :as set]
            [clojure.pprint :as pp]
            [me.raynes.fs :as fs]
            [stencil.core :as stencil]
            [boot.user]
            [boot.pod :as pod]
            [boot.core :as boot]
            [boot.util :as util]
            [boot.task.built-in :as builtin]
            [boot.file            :as file]
            [boot.from.digest     :as digest])
  (:import [com.google.appengine.tools KickStart]
           [java.io File]
           [java.net URL URLClassLoader]))

            ;; [deraen.boot-less.version :refer [+version+]]))

(def boot-version "2.7.1")

(def web-xml-edn "web.xml.edn")
(def webapp-edn "webapp.edn")
(def gae-edn "appengine.edn")
(def appstats-edn "appstats.edn")

;; for microservices app:
(def services-edn "services.edn")


(def meta-inf-dir "META-INF")
(def web-inf-dir "WEB-INF")
  ;; (let [mod (-> (boot/get-env) :gae :module :name)]
  ;;   (str (if mod (str mod "/")) "WEB-INF")))

;; http://stackoverflow.com/questions/2751033/clojure-program-reading-its-own-manifest-mf
(defn manifest-map
  "Returns the mainAttributes of the manifest of the passed in class as a map."
  [j]
  (println "JAR: " j (type j))
  (->> (str "jar:file:" j "!/META-INF/MANIFEST.MF")
       clojure.java.io/input-stream
       java.util.jar.Manifest.
       .getMainAttributes
       (map (fn [[k v]] [(str k) v]))
       (into {})))

(defn expand-home [s]
  (if (or (.startsWith s "~") (.startsWith s "$HOME"))
    (str/replace-first s "~" (System/getProperty "user.home"))
    s))

(def sdk-string (let [jars (pod/resolve-dependency-jars (boot/get-env) true)
                      zip (filter #(.startsWith (.getName %) "appengine-java-sdk") jars)]
                  (if (empty? zip)
                    (println "appengine-java-sdk zipfile not found")
                    (let [fname (first (for [f zip] (.getName f)))
                          sdk-string (subs fname 0 (.lastIndexOf fname "."))]
                      ;; (println "sdk-string: " sdk-string)
                      sdk-string))))

(def config-map (merge {:build-dir "target"
                        :sdk-root (let [dir (str (System/getenv "HOME") "/.appengine-sdk")]
                                    (str dir "/" sdk-string))}
                       (reduce (fn [altered-map [k v]] (assoc altered-map k
                                                              (if (= k :sdk-root)
                                                                (str (expand-home v) "/" sdk-string)
                                                                v)))
                               {}
                               (:gae (boot/get-env)))))

;;(str (:build-dir config-map)

(def classes-dir (str web-inf-dir "/classes"))

(def lib-dir (str web-inf-dir "/lib"))

(def appcfg-class "com.google.appengine.tools.admin.AppCfg")

;; for multiple subprojects:
(def root-dir "")
(def root-project "")

(def project-dir (System/getProperty "user.dir"))
(def build-dir (str/join "/" [project-dir "target"]))

(defn output-libs-dir [nm] (str/join "/" [build-dir "libs"]))
(defn output-resources-dir [nm] (str/join "/" [build-dir "resources" nm]))

(defn java-source-dir [nm] (str/join "/" [project-dir "src" nm "java"]))
(defn input-resources-dir [nm] (str/join "/" [project-dir "src" nm "resources"]))

(defn gae-app-dir []
  (let [mod (str (-> (boot/get-env) :gae :module :name))]
    (str "target" (if mod (str "/" mod)))))

(def sdk-root-property "appengine.sdk.root")
(def java-classpath-sys-prop-key "java.class.path")
(def sdk-root-sys-prop-key "appengine.sdk.root")

(defn print-task
  [task opts]
  (if (or (:verbose opts) (:verbose config-map) (:list-tasks config-map))
    (println "TASK: " task)))

(defn dump-props []
  (println "project-dir: " project-dir)
  (println "build-dir: " build-dir)
  (println "classes-dir: " classes-dir)
  (println "output-resources-dir: " (output-resources-dir nil))
  (println "java-source-dir: " (java-source-dir nil))
  (println "input-resources-dir: " (input-resources-dir nil))
  )

(defn dump-env []
  (let [e (boot/get-env)]
    (println "ENV:")
    (util/pp* e)))


(defn dump-tmpfiles
  [lbl tfs]
  (println "\n" lbl ":")
  (doseq [tf tfs] (println (boot/tmp-file tf))))

(defn dump-tmpdirs
  [lbl tds]
  (println "\n" lbl ":")
  (doseq [td tds] (println td)))

(defn dump-fs
  [fileset]
  (doseq [f (:dirs fileset)] (println "DIR: " f))
  (doseq [f (:tree fileset)] (println "F: " f)))
  ;; (dump-tmpfiles "INPUTFILES" (boot/input-files fileset))
  ;; (dump-tmpdirs "INPUTDIRS" (boot/input-dirs fileset))
  ;; (dump-tmpdirs "OUTPUTFILESET" (boot/output-files (boot/output-fileset fileset)))
  ;; (dump-tmpfiles "OUTPUTFILES" (boot/output-files fileset))
  ;; (dump-tmpdirs "OUTPUTDIRS" (boot/output-dirs fileset))
  ;; (dump-tmpfiles "USERFILES" (boot/user-files fileset))
  ;; (dump-tmpdirs "USERDIRS" (boot/user-dirs fileset)))

(def config-props
  {;; https://docs.gradle.org/current/userguide/war_plugin.html
   :web-app-dir-name "src/main/webapp" ;; String
   ;; :web-app-dir "project-dir/web-app-dir-name" ;; File
   })

#_(def runtask-params-defaults
  {;; :http-address 127.0.0.1
   ;; :http-port 8080
   ;; :daemon false
   ;; :disable-update-check false
   ;; :disable-datagram false
   ;; :jvm-flags []
   :allow-remote-shutdown true})
   ;; :download-sdk false

(def kw->opt
  {
   :allow-remote-shutdown "--allow_remote_shutdown"
   :default-gcs-bucket "--default_gcs_bucket"
   :disable-filesapi-warning "-disable_filesapi_warning"
   :disable_restricted_check "--disable_restricted_check"
   :disable-update-check "--disable_update_check"
   :enable_filesapi "--enable_filesapi"
   :enable-jacoco "--enable_jacoco"
   :jacoco-agent-jar "--jacoco_agent_jar"
   :jacoco-agent-args "--jacoco_agent_args"
   :jacoco-exec "--jacoco_exec"
   :external-resource-dir "--external_resource_dir"
   :generated-war-dir "--generated_dir" ;; "Set the directory where generated files are created."
   :generate-war "--generate_war"
   :http-address "--address"
   :http-port "--port"
   :instance-port "--instance_port"
   :jvm-flags "--jvm_flags"
   :no-java-agent "--no_java_agent"
   :property "--property" ;; ????
   :sdk-server "--server"  ;; DevAppServer param
   :sdk-root "--sdk_root"
   :start-on-first-thread "--startOnFirstThread"})

(defn ->args [param-map]
  (let [r (flatten (for [[k v] param-map]
                      (if (= k :jvm-flags)
                        (let [flags (str/split (first v) #" ")
                              fargs (into []
                                          (for [flag flags] (str "--jvm-flag=\"" flag "\"")))]
                          (do ;(println "FLAGS: " flags (type flags) (type (first flags)))
                              ;(println "FARGS: " fargs (type fargs))
                              (seq fargs)))
                        (str (get kw->opt k) "=" v))))]
    #_(println "MERGE: " (pr-str r))
    r))

(defn- find-mainfiles [fs]
  (->> fs
       boot/input-files
       (boot/by-ext [".clj"])))


(defn get-tools-jar []
  (let [file-sep (System/getProperty "file.separator")
        ;; _ (println "File Sep: " file-sep)
        tools-api-jar (str/join file-sep [(:sdk-root config-map) "lib" "appengine-tools-api.jar"])]
    (if (not (.exists (io/as-file tools-api-jar)))
      (throw (Exception. (str "Required library 'appengine-tools-api.jar' could not be found in specified path: " tools-api-jar "!"))))
    tools-api-jar))

(defn validate-tools-api-jar []
  (let [tools-api-jar (get-tools-jar)
        path-sep (File/pathSeparator)
        jcp (System/getProperty java-classpath-sys-prop-key)]
    #_(if (not (.contains jcp tools-api-jar))
        (System/setProperty java-classpath-sys-prop-key (str/join path-sep [jcp tools-api-jar])))
    (System/setProperty java-classpath-sys-prop-key tools-api-jar)
    #_(println "Java classpath: " (System/getProperty java-classpath-sys-prop-key))

    ;; Adding appengine-tools-api.jar to context ClassLoader
    ;; see also http://stackoverflow.com/questions/194698/how-to-load-a-jar-file-at-runtime?lq=1
    (let [;; ClassLoader rootClassLoader = ClassLoader.systemClassLoader.parent
          root-class-loader (.getParent (ClassLoader/getSystemClassLoader))
          ;;URLClassLoader appengineClassloader
          ;;  = new URLClassLoader([new File(appEngineToolsApiJar).toURI().toURL()] as URL[], rootClassLoader)
          gae-class-loader (let [tools-jar-url [(.toURL (.toURI (io/as-file tools-api-jar)))]]
                                 (URLClassLoader. (into-array tools-jar-url) root-class-loader))]
      ;; Thread.currentThread().setContextClassLoader(appengineClassloader)
      (.setContextClassLoader (Thread/currentThread) gae-class-loader))))

;; (boot/deftask foo "" []
;;   (boot/with-pre-wrap [fs]
;;     (println "FS: " (type fs))
;;     ;; (let [asset-files (->> fs boot/output-files (boot/by-ext [".clj"]))]
;;     ;;      (doseq [asset asset-files] (println "foo" asset)))
;;     fs))

;; (boot/deftask clj
;;   "Assetize and mv source .clj files to <build-dir>/WEB-INF/classes"
;;   []
;;   (builtin/sift :move {#"(.*\.clj$)" (str web-inf-dir "/classes/$1")}
;;                 :to-resource #{#"clj$"}))

;; (boot/deftask aot
;;   "Built-in aot does not allow custom *compile-path*"

;;   [a all          bool   "Compile all namespaces."
;;    d dir PATH str "where to place generated class files"
;;    n namespace NS #{sym} "The set of namespaces to compile."]

;;   (let [tgt         (boot/tmp-dir!)
;;         dir         (if dir dir "")
;;         out-dir     (io/file tgt dir)
;;         foo         (doto (io/file out-dir "foo") io/make-parents)
;;         pod-env     (update-in (boot/get-env) [:directories] conj (.getPath out-dir))
;;         compile-pod (future (pod/make-pod pod-env))]
;;     (boot/with-pre-wrap [fs]
;;       (boot/empty-dir! tgt)
;;       (let [all-nses (->> fs boot/fileset-namespaces)
;;             nses     (->> all-nses (set/intersection (if all all-nses namespace)) sort)]
;;         (pod/with-eval-in @compile-pod
;;           (require '[clojure.java.io :as io])
;;           (let [foo ~(.getPath foo)]
;;             (io/make-parents (io/as-file foo))
;;             ;;(pod/add-classpath (io/as-file outdir))
;;             (binding [*compile-path* ~(.getPath out-dir)]
;;               (doseq [[idx ns] (map-indexed vector '~nses)]
;;                 (boot.util/info "Compiling %s/%s %s...\n" (inc idx) (count '~nses) ns)
;;                 (compile ns)))))
;;         (-> fs (boot/add-resource tgt) boot/commit!)))))

(boot/deftask appengine
  "generate gae appengine-web.xml"
  [c config-syms SYMS #{sym} "namespaced symbols bound to meta-config data"
   d dir DIR str "output dir"
   k keep bool str "keep intermediate .clj files"
   m module MODULE str "module dirpath"
   v verbose bool "Print trace messages."]
  (let [tmp-dir (boot/tmp-dir!)
        prev-pre (atom nil)
        dir (if dir dir web-inf-dir)]
    (boot/with-pre-wrap fileset
      (let [webapp-edn-files (->> (boot/fileset-diff @prev-pre fileset)
                                  boot/input-files
                                  (boot/by-name [webapp-edn]))
            webapp-edn-f (condp = (count webapp-edn-files)
                           0 nil
                           1 (first webapp-edn-files)
                           (throw (Exception. "only one webapp.edn file allowed")))
            webapp-edn-c (if webapp-edn-f (-> (boot/tmp-file webapp-edn-f) slurp read-string) {})
            gae-edn-fs (->> (boot/fileset-diff @prev-pre fileset)
                            boot/input-files
                            (boot/by-name [gae-edn]))]
        (if (> (count gae-edn-fs) 1) (throw (Exception. "only one web.xml.edn file allowed")))
        (if (= (count gae-edn-fs) 0) (throw (Exception. "web.xml.edn file not found")))

        (let [gae-edn-f (first gae-edn-fs)
              gae-forms (-> (boot/tmp-file gae-edn-f) slurp read-string)
              gae-forms (merge gae-forms
                               (if (-> webapp-edn-c :module)
                                 (if (-> webapp-edn-c :module :default)
                                   {}
                                   {:module (str (-> webapp-edn-c :module :name))})
                                 {}))
              app-id (name (-> (boot/get-env) :gae :app-id))
              version (str/replace (-> (boot/get-env) :gae :version) "." "-")
              version (str "r-" (subs version 0 (.lastIndexOf version "-SNAPSHOT")))
              module (-> (boot/get-env) :gae :module :name)
              ;; module (if (:module gae-forms)
              ;;          (str (:module gae-forms) "-" version)
              ;;          nil)
              gae-map (assoc gae-forms
                             :app-id app-id
                             :version version
                             :module {:name module})]
          ;; (println "gae-map: " gae-map)
          (let [content (stencil/render-file
                         "migae/boot_gae/xml.appengine-web.mustache"
                         gae-map)]
            (if verbose (println content))
            (util/info "Configuring appengine-web.xml\n")
            (let [xml-out-path (str dir "/appengine-web.xml")
                  xml-out-file (doto (io/file tmp-dir xml-out-path) io/make-parents)
                  ]
              (spit xml-out-file content)))))
      (-> fileset (boot/add-resource tmp-dir) boot/commit!))))

;; (defn- add-appstats!
;;   [reloader-ns urls in-file out-file]
;;   (let [spec (-> in-file slurp read-string)]
;;     (util/info "Adding :appstats to %s...\n" (.getName in-file))
;;     (io/make-parents out-file)
;;     (let [m (-> spec
;;                 (assoc-in [:reloader]
;;                           {:ns reloader-ns
;;                            :name "reloader"
;;                            :display {:name "Clojure reload filter"}
;;                            :urls (if (empty? urls) [{:url "./*"}]
;;                                      (merge (for [url urls] {:url url})))
;;                            :desc {:text "Clojure reload filter"}}))
;;           s (with-out-str (pp/pprint m))]
;;     (spit out-file s))))

(boot/deftask appstats
  "enable GAE Appstats"
  [k keep bool "keep intermediate .clj files"
   ;; n gen-reloader-ns NS sym "ns for gen-reloader"
   ;; r reloader-impl-ns NS sym "ns for reloader"
   w web-inf WEB-INF str "WEB-INF dir, default: WEB-INF"
   v verbose bool "Print trace messages."]
  (let [edn-tmp (boot/tmp-dir!)
        prev-pre (atom nil)
        web-inf (if web-inf web-inf web-inf-dir)

        ;; gen-reloader-ns (if gen-reloader-ns (symbol gen-reloader-ns) (gensym "reloadergen"))
        ;; gen-reloader-path (str gen-reloader-ns ".clj")
        ]
    (comp
     (boot/with-pre-wrap [fileset]
       (let [web-xml-edn-files (->> (boot/user-files fileset)
                                    (boot/by-re [(re-pattern (str web-xml-edn "$"))]))
                    ;; (boot/by-name [web-xml-edn]))
             web-xml-edn-f (condp = (count web-xml-edn-files)
                             0 (do (util/info (str "Creating " web-xml-edn "\n"))
                                   nil)
                             1 (first web-xml-edn-files)
                             (throw (Exception. "only one web.xml.edn file allowed")))
             web-xml-edn-c (if web-xml-edn-f (-> (boot/tmp-file web-xml-edn-f) slurp read-string) {})]
         ;; (println "web-xml-edn-c: " web-xml-edn-c)
           (if (:appstats web-xml-edn-c)
             fileset
             (do
               ;; (add-appstats! reloader-impl-ns urls in-file out-file)
               (let [appstats-fs (->> (boot/input-files fileset)
                                      (boot/by-name [appstats-edn]))]
                 (if (> (count appstats-fs) 1)
                   (throw (Exception. (str "only one " appstats-edn " file allowed"))))
                 (if (= (count appstats-fs) 0)
                   (throw (Exception. (str appstats-edn " file not found"))))
                 (let [appstats-edn-f (first appstats-fs)
                       appstats-config (-> (boot/tmp-file appstats-edn-f) slurp read-string)]
                   (util/info (str "Elaborating " web-xml-edn " with :appstats stanza\n"))
                   (let [m (-> web-xml-edn-c
                               (assoc-in [:appstats] (:appstats appstats-config)))
                         web-xml-edn-s (with-out-str (pp/pprint m))
                         web-xml-edn-out-file (io/file edn-tmp (boot/tmp-path web-xml-edn-f))]
                     (io/make-parents web-xml-edn-out-file)
                     (spit web-xml-edn-out-file web-xml-edn-s)))))))
               (reset! prev-pre
                       (-> fileset
                           (boot/add-source edn-tmp)
                           boot/commit!))))))

(declare filters install-sdk libs logging reloader servlets target webxml)

(boot/deftask assemble
  "assemble a service-based app (ear)"
  [v verbose bool "Print trace messages."]
  (let [tmpdir (boot/tmp-dir!)
        dfl-scopes #{"compile" "runtime" "provided"}
        scopes dfl-scopes
        ;; scopes     (-> dfl-scopes
        ;;                (set/union include-scope)
        ;;                (set/difference exclude-scope))
        scope?     #(contains? scopes (:scope (util/dep-as-map %)))
        checkouts  (->> (boot/get-checkouts)
                        (filter (comp scope? :dep val))
                        (into {}))
        co-jars    (->> checkouts (map (comp :jar val)))
        co-dirs    (->> checkouts (map (comp :dir val)))

        checkout-vec (-> (boot/get-env) :checkouts)
        cos (map #(assoc (apply hash-map %) :coords [(first %) (second %)]) checkout-vec)
        ]
    (boot/with-pre-wrap [fileset]
      (doseq [co cos]
        (let [mod (if (:default co) "default" (if-let [mod (:module co)] mod "default"))]
          (let [corec (get checkouts (first (:coords co)))
                co-dir (:dir corec)
                jar-path (.getPath (:jar corec))
                out-dir (doto (io/file tmpdir mod) io/make-parents)]
            ;; (println "co-dir: " co-dir)
            ;; (println "out-dir: " out-dir)
            (fs/copy-dir co-dir out-dir))))
      (-> fileset
          (boot/add-asset tmpdir :exclude #{(re-pattern (str "/META-INF/.*"))})
          (boot/commit!)))))

(boot/deftask build
  "assemble, configure, and build app"
  [k keep bool "keep intermediate .clj and .edn files"
   p prod bool "production build, without reloader"
   s service bool "build a service"
   v verbose bool "verbose"]
  (let [keep (or keep false)
        verbose (or verbose false)]
        ;; mod (str (-> (boot/get-env) :gae :module :name))]
    ;; (println "MODULE: " mod)
    (comp (install-sdk)
          (libs :verbose verbose)
          (appstats :verbose verbose)
          (builtin/javac)
          (if prod identity (reloader :keep keep :service service :verbose verbose))
          (filters :keep keep :verbose verbose)
          (servlets :keep keep :verbose verbose)
          (logging :verbose verbose)
          (webxml :verbose verbose)
          (appengine :verbose verbose)
          (builtin/sift :move {#"(.*clj$)" (str classes-dir "/$1")})
          (builtin/sift :move {#"(.*\.class$)" (str classes-dir "/$1")})
          (if service
            (comp
             (builtin/pom)
             (builtin/jar))
            identity)
          (target :service service :verbose verbose)
          #_(builtin/sift :include #{(re-pattern (str mod ".*jar"))})
          #_(builtin/install)
          #_(target :verbose verbose)
          ;; (builtin/sift :move {#"(^.*)" (str mod "/$1")})
          ;; FIXME: use cache instead?
          ;; (builtin/target :dir #{(str "target/" mod)})
          )))

(boot/deftask target
  "target, using module name"
  [C no-clean bool "Don't clean target before writing project files"
   m monitor bool "monitoring - target service assembly"
   s service bool "building as service"
   v verbose bool "verbose"]
  (let [mod      (-> (boot/get-env) :gae :module :name)
        app-dir  (-> (boot/get-env) :gae :module :app-dir)
        dir      (if monitor
                   (str app-dir "/target/" mod)
                   (if service (str "target/" mod)
                       "target"))]
    (if service
      (if (not (and mod app-dir))
        (throw (Exception. "For service targets, both :name and :app-dir must be specified in :gae map of build.boot"))))
    (println "TARGET DIR: " dir)
    (builtin/target :dir #{dir}
                    :no-clean (or no-clean false))))

(boot/deftask cache
  "control cache: retrieve, save, clean"
  [c clean bool "clean config - clear cache first"
   t trace bool "dump cache to stdout if verbose=true"
   r retrieve bool "retrieve cached master edn config file and sync to fileset (default)"
   s save bool "save master edn config file from fileset to cache"
   v verbose bool "verbose"]
  ;; default to retrieve
  (if (and retrieve save)
    (util/exit-error
     (util/fail "boot-gae/cache: only one of :retrieve and :save may be specified\n")))
  (let [retrieve (or retrieve (not save))
        save (or (not retrieve) save)]
    (boot/with-pre-wrap fileset
      (let [cache-dir (boot/cache-dir! :boot-gae/build)]
        (if verbose (println "cache-dir: " (.getPath cache-dir)))
        (if clean
          (do (if verbose (util/info (str "Clearing boot-gae cache\n")))
              (boot/empty-dir! cache-dir)
              (boot/commit! fileset))
          (if retrieve
            (do (println "RETRIEVING " (type cache-dir))
                (let [;;fs (boot/add-cached-asset fileset (io/file "boot-gae/build") identity)
                      fs (boot/add-asset fileset cache-dir)]
                  #_(println "FS: " fs)
                  (boot/commit! fs)))
            (if save
              (let [_ (println "SAVING")
                    fs (into [] (boot/output-dirs fileset))]
                ;;(println "fs: " fs (type fs))
                (apply boot/sync! cache-dir fs)
                fileset)
              fileset)))))))

(boot/deftask earxml
  "generate xml config files for microservices app"
  [d dir DIR str "output dir"
   c config-syms SYMS #{sym} "namespaced symbols bound to meta-config data"
   k keep bool str "keep intermediate .clj files"
   v verbose bool "Print trace messages."]
  (let [edn-tmp (boot/tmp-dir!)
        prev-pre (atom nil)
        odir (if dir dir meta-inf-dir)]
    ;; FIXME create services.edn map from :checkouts
    (boot/with-pre-wrap fileset
      (let [services-fs (->> (boot/fileset-diff @prev-pre fileset)
                                boot/input-files
                                (boot/by-re [(re-pattern (str services-edn "$"))]))]
            ;; (boot/by-name [services]))]
        (if (> (count services-fs) 1) (throw (Exception. "only one services.edn file allowed")))
        (if (= (count services-fs) 0) (throw (Exception. "services.edn file not found")))

        (let [services-f (first services-fs)
              services-cfg (-> (boot/tmp-file services-f) slurp read-string)
              path     (boot/tmp-path services-f)
              in-file  (boot/tmp-file services-f)
              out-file (io/file edn-tmp path)]
          (let [appengine-application-cfg (stencil/render-file
                                           "migae/boot_gae/xml.appengine-application.mustache"
                                           services-cfg)
                application-cfg (stencil/render-file
                                 "migae/boot_gae/xml.application.mustache"
                                 services-cfg)
                manifest-cfg (stencil/render-file
                              "migae/boot_gae/MANIFEST.MF.mustache"
                              services-cfg)]
            (if verbose
              (do (println appengine-application-cfg)
                  (println application-cfg)
                  (println manifest-cfg)))
            (let [appengine-application-out-path (str odir "/appengine-application.xml")
                  appengine-application-out-file (doto
                                                     (io/file edn-tmp appengine-application-out-path)
                                                   io/make-parents)
                  application-out-path (str odir "/application.xml")
                  application-out-file (doto (io/file edn-tmp application-out-path)
                                         io/make-parents)
                  manifest-out-path (str odir "/MANIFEST.MF")
                  manifest-out-file (doto (io/file edn-tmp manifest-out-path)
                                      io/make-parents)
                  ]
              (util/info "Configuring microservices app\n")
              (spit appengine-application-out-file appengine-application-cfg)
              (spit application-out-file application-cfg)
              (spit manifest-out-file manifest-cfg)))))
      (-> fileset (boot/add-resource edn-tmp) boot/commit!))))

(boot/deftask deploy
  "Installs a new version of the application onto the server, as the default version for end users."
  ;; options from AppCfg.java, see also appcfg.sh --help
  [s server SERVER str "--server"
   e email EMAIL str   "The username to use. Will prompt if omitted."
   H host  HOST  str   "Overrides the Host header sent with all RPCs."
   p proxy PROXYHOST str "'PROXYHOST[:PORT]'.  Proxies requests through the given proxy server."
   _ proxy-https PROXYHOST str "'PROXYHOST[:PORT].  Proxies HTTPS requests through the given proxy server."
   _ no-cookies bool      "Do not save/load access credentials to/from disk."
   _ sdk-root ROOT str   "Overrides where the SDK is located."
   b build-dir BUILD str "app build dir"
   _ passin   bool          "Always read the login password from stdin."
   A application APPID str "application id"
   M module MODULE str      "module"
   V version VERSION str   "(major) version"
   _ oauth2 bool            "Ignored (OAuth2 is the default)."
   _ noisy bool             "Log much more information about what the tool is doing."
   _ enable-jar-splitting bool "Split large jar files (> 10M) into smaller fragments."
   _ jar-splitting-excludes SUFFIXES str "list of files to be excluded from all jars"
   _ disable-jar-jsps bool "Do not jar the classes generated from JSPs."
   _ enable-jar-classes bool "Jar the WEB-INF/classes content."
   _ delete-jsps bool "Delete the JSP source files after compilation."
   _ retain-upload-dir bool "Do not delete temporary (staging) directory used in uploading."
   _ compile-encoding ENC str "The character encoding to use when compiling JSPs."
   _ num-days NUM_DAYS int "number of days worth of log data to get"
   _ severity SEVERITY int "Severity of app-level log messages to get"
   _ include-all bool   "Include everything in log messages."
   a append bool          "Append to existing file."
   _ num-runs NUM-RUNS int "Number of scheduled execution times to compute."
   f force bool           "Force deletion of indexes without being prompted."
   _ no-usage-reporting bool "Disable usage reporting."
   D auto-update-dispatch bool "Include dispatch.yaml in updates"
   _ sdk-help bool "Display SDK help screen"
   v verbose bool          "Print invocation args"]
  (if verbose (println "TASK: boot-gae/deploy"))
  (validate-tools-api-jar)
  (println "PARAMS: " *opts*)
  (let [opts (merge {:sdk-root (:sdk-root config-map)
                     ;; :use-java7 true
                     :build-dir (gae-app-dir)} ;; (:build-dir config-map)}
                    *opts*)
        _ (println "OPTS: " opts)
        params (into [] (for [[k v] (remove (comp nil? second)
                                            (dissoc opts :build-dir :verbose :sdk-help))]
                          (str "--" (str/replace (name k) #"-" "_")
                               (if (not (instance? Boolean v)) (str "=" v)))))
        params (if (:sdk-help *opts*)
                 ["help" "update"]
                 (conj params "update" (:build-dir opts)))
        params (into-array String params)
        ;; ClassLoader classLoader = Thread.currentThread().contextClassLoader
        class-loader (-> (Thread/currentThread) (.getContextClassLoader))
        cl (.getParent class-loader)
        app-cfg (Class/forName appcfg-class true class-loader)]

  ;; def appCfg = classLoader.loadClass(APPENGINE_TOOLS_MAIN)
  ;; appCfg.main(params as String[])
    (def method (first (filter #(= (. % getName) "main") (. app-cfg getMethods))))
    (def invoke-args (into-array Object [params]))
    (if verbose ;; (or (:verbose *opts*) (:verbose config-map))
      (do (println "CMD: AppCfg")
          (doseq [arg params]
            (println "\t" arg))))
    (. method invoke nil invoke-args)))

(boot/deftask filters
  "generate filters; update web.xml.edn with filter config data"

  [k keep bool "keep intermediate .clj files"
   n gen-filters-ns NS str "namespace to generate and aot; default: 'filters"
   w web-inf WEB-INF str "WEB-INF dir, default: WEB-INF"
   v verbose bool "Print trace messages."]
  (let [edn-tmp-dir (boot/tmp-dir!)
        prev-pre (atom nil)
        ;; config-sym (if config-sym config-sym 'filters/config)
        ;; config-ns (symbol (namespace config-sym))
        web-inf (if web-inf web-inf web-inf-dir)
        filters-edn "filters.edn"
        gen-filters-tmp-dir (boot/tmp-dir!)
        gen-filters-ns (if gen-filters-ns (symbol gen-filters-ns) (gensym "filtersgen"))
        gen-filters-path (str gen-filters-ns ".clj")]
    (comp
     (boot/with-pre-wrap [fileset]
       (let [web-xml-edn-files (->> (boot/fileset-diff @prev-pre fileset)
                                    boot/input-files
                                    (boot/by-re [(re-pattern (str web-xml-edn "$"))]))]
                    ;; (boot/by-name [web-xml-edn]))]
         (if (> (count web-xml-edn-files) 1)
           (throw (Exception. "only one web.xml.edn file allowed")))
         (if (= (count web-xml-edn-files) 0)
           (throw (Exception. "cannot find web.xml.edn")))

         (let [web-xml-edn-f (first web-xml-edn-files)
               web-xml-edn-c (-> (boot/tmp-file web-xml-edn-f) slurp read-string)]
           (if (:filters web-xml-edn-c)
             fileset
             (do
               ;; step 0: read the edn files
               (let [filters-edn-files (->> (boot/input-files fileset)
                                            (boot/by-name [filters-edn]))]
                 (if (> (count filters-edn-files) 1)
                   (throw (Exception. "only one filters.edn file allowed")))
                 (if (= (count filters-edn-files) 0)
                   (throw (Exception. "cannot find filters.edn")))
                 (let [edn-filters-f (first filters-edn-files)
                       filter-configs (-> (boot/tmp-file edn-filters-f) slurp read-string)
                       smap (-> web-xml-edn-c (assoc-in [:filters]
                                                        (:filters filter-configs)))
                       web-xml-edn-s (with-out-str (pp/pprint smap))]
                   ;; step 1:  inject filter config stanza to web.xml.edn
                   (let [path     (boot/tmp-path web-xml-edn-f)
                         web-xml-edn-in-file  (boot/tmp-file web-xml-edn-f)
                         web-xml-edn-out-file (io/file edn-tmp-dir path)]
                     (io/make-parents web-xml-edn-out-file)
                     (spit web-xml-edn-out-file web-xml-edn-s))

                   ;; step 2: gen filters
                   (let [gen-filters-content (stencil/render-file "migae/boot_gae/gen-filters.mustache"
                                                                   (assoc filter-configs
                                                                          :gen-filters-ns
                                                                          gen-filters-ns))
                         gen-filters-out-file (doto
                                                   (io/file gen-filters-tmp-dir gen-filters-path)
                                                 io/make-parents)]
                     (spit gen-filters-out-file gen-filters-content))))))))
       (util/info "Configuring filters...\n")

       ;; step 3: commit files to fileset
       (reset! prev-pre
               (-> fileset
                   (boot/add-source edn-tmp-dir)
                   (boot/add-source gen-filters-tmp-dir)
                   boot/commit!)))
     (builtin/aot :namespace #{gen-filters-ns}) ;; :dir (str web-inf "/classes"))
     (if keep identity
       (builtin/sift :include #{(re-pattern (str gen-filters-ns ".*.class"))}
                     :invert true))
     (if keep
       (comp
        (builtin/sift :to-resource #{(re-pattern web-xml-edn)})
        (builtin/sift :to-asset #{(re-pattern gen-filters-path)}))
       identity)
     )))

(boot/deftask install-sdk
  "Unpack and install the SDK zipfile"
  [r release SDKREL str "SDK verion"
   v verbose bool "Print trace messages"]
  ;;NB: java property expected by kickstart is "appengine.sdk.root"
  ;; (print-task "install-sdk" *opts*)
  (let [release (or release "RELEASE")
        coords (vector 'com.google.appengine/appengine-java-sdk
                       "RELEASE"
                       :extension "zip")
        jar-path (pod/resolve-dependency-jar (boot/get-env) coords)
        sdk-dir (io/as-file (:sdk-root config-map))
        prev        (atom nil)]
    (boot/with-pre-wrap fileset
      (if (.exists sdk-dir)
        (do
          (let [file-sep (System/getProperty "file.separator")
                tools-api-jar (str/join file-sep [(:sdk-root config-map) "lib" "appengine-tools-api.jar"])]
            (if (not (.exists (io/as-file tools-api-jar)))
              (do
                (println (str "Found sdk-dir " sdk-dir " but not its contents; re-exploding"))
                (boot/empty-dir! sdk-dir)
                (println "Exploding SDK\n from: " jar-path "\n to: " (.getPath sdk-dir))
                (pod/unpack-jar jar-path (.getParent sdk-dir)))
              (if (or (:verbose *opts*) (:verbose config-map))
                (println "SDK already installed at: " (.getPath sdk-dir))))))
        (do
          (if (or (:verbose *opts*) (:verbose config-map))
            (println "Installing unpacked SDK to: " (.getPath sdk-dir)))
          (pod/unpack-jar jar-path (.getParent sdk-dir))))
      fileset)))

;;(core/deftask buber
;; FIXME: what was this for???
#_(defn buber
  [tgt]
  (println "BUBER ")
  (let [;; tgt        (boot/tmp-dir!)
        cache      (boot/cache-dir! ::uber :global true)
        dfl-scopes #{"compile" "runtime"}
        scopes     dfl-scopes
        ;; scopes     (-> dfl-scopes
        ;;                (set/union include-scope)
        ;;                (set/difference exclude-scope))
        _ (println "SCOPES: " dfl-scopes)
        scope?     #(contains? scopes (:scope (util/dep-as-map %)))
        jars       (-> pod/env ;; (boot/get-env)
                       (update-in [:dependencies] (partial filter scope?))
                       pod/resolve-dependency-jars)
        _ (println "JARS: " jars)
        jars       (remove #(.endsWith (.getName %) ".pom") jars)
        checkouts  (->> (boot/get-checkouts)
                        (filter (comp scope? :dep val))
                        (into {}))
        co-jars    (->> checkouts (map (comp :jar val)))
        ;; co-dirs    (->> checkouts (map (comp :dir val)))
        ;; exclude    (or exclude pod/standard-jar-exclusions)
        ;; merge      (or merge pod/standard-jar-mergers)
        ;; reducer    (fn [xs jar]
        ;;              (boot/add-cached-resource
        ;;                xs (digest/md5 jar) (partial pod/unpack-jar jar)
        ;;                :include include :exclude exclude :mergers merge))
        ;; co-reducer #(boot/add-resource
        ;;               %1 %2 :include include :exclude exclude :mergers merge)
        ]
    ;; (boot/with-pre-wrap [fs]
    ;; (println "co jars: " co-jars)
      (when (seq jars)
        (doseq [jar jars] ;; (reduce into [] [jars co-jars])]
          (let [dest (doto (io/file tgt (.getName jar)) io/make-parents)]
            (println "dest: " dest)
            (file/copy-atomically jar dest))))))

      ;;       (when-not (.exists src)
      ;;         (println "CACHING " name)
      ;;         (util/dbug* "Caching jar %s...\n" name)
      ;;         (file/copy-atomically jar src))
      ;;       (util/dbug* "Adding cached jar %s...\n" name)
      ;;       (println (format "Adding cached jar %s...\n" name))
      ;;       (file/hard-link src (io/file tgt name)))))
      ;; #_(boot/commit! (if as-jars
      ;;                 (boot/add-resource fs tgt)
      ;;                 (reduce co-reducer (reduce reducer fs jars) co-dirs)))))

;; FIXME:  for dev phase, we want to include deps in test scope
(boot/deftask libs
  ""
  [v verbose bool "Print trace messages."]
  (let [checkout-vec (-> (boot/get-env) :checkouts)
        cos (map #(assoc (apply hash-map %) :coords [(first %) (second %)]) checkout-vec)

        dfl-scopes #{"compile" "runtime" "provided"}
        scopes dfl-scopes
        ;; scopes     (-> dfl-scopes
        ;;                (set/union include-scope)
        ;;                (set/difference exclude-scope))
        scope?     #(contains? scopes (:scope (util/dep-as-map %)))
        checkouts  (->> (boot/get-checkouts)
                        (filter (comp scope? :dep val))
                        (into {}))
        cache      (boot/cache-dir! ::uber :global true)
        ]
    (if (empty? cos)
      (do
        ;;(println "NO CHECKOUTS")
          (comp
           (builtin/uber :as-jars true :exclude-scope #{"provided"})
           ;; (builtin/sift :include #{#"zip$"} :invert true)
           ;; (builtin/sift :include #{#".*appengine-api-.*jar$"} :invert true)
           ;; (builtin/sift :include #{#".*jar$"})
           (builtin/sift :move {#"(.*\.jar$)" (str lib-dir "/$1")})))
      (boot/with-pre-wrap [fileset]
        ;;(println "CHECKOUTS")
        (let [tmpdir     (boot/tmp-dir!)]
          (doseq [co cos]
            (let [mod (if (:default co) "default"
                          (if-let [mod (:module co)]
                            mod "default"))
                  coords (:coords co)
                  pod-env (update-in (dissoc (boot/get-env) :checkouts)
                                     [:dependencies] #(identity %2)
                                     (concat '[[boot/core "2.7.1"] ;; ~(str boot-version)]
                                               [boot/pod "2.7.1"]] ;; ~(str boot-version)]]
                                             (vector coords)))
                  pod (future (pod/make-pod pod-env))]
              (if verbose (println "MODULE: " mod))
              (pod/with-eval-in @pod
                (require '[boot.pod :as pod] '[boot.util :as util]
                         '[boot.core :as boot] '[boot.file :as file]
                         '[boot.task.built-in :as builtin] '[boot.from.digest :as digest]
                         '[clojure.java.io :as io])
                (let [dfl-scopes #{"compile" "runtime" "test"}
                      scopes dfl-scopes
                      scope? #(contains? scopes (:scope (util/dep-as-map %)))
                      co-jar (pod/resolve-dependency-jar (boot/get-env) '~coords)
                      jars   (remove #(= (str co-jar) (.getPath %))
                                     (-> pod/env
                                         (update-in [:dependencies] (partial filter scope?))
                                         (pod/resolve-dependency-jars)))]
                  (doseq [jar jars]
                    (let [hash (digest/md5 jar)
                          name (str hash "-" (.getName jar))
                          tmpjar-path (str ~mod "/WEB-INF/lib/" (.getName jar))
                          cached-jar  (io/file ~(.getPath cache) hash)]
                      (when-not (.exists cached-jar)
                        (if ~verbose (util/info "Caching jar %s...\n" name))
                        (file/copy-atomically jar cached-jar))
                      (if ~verbose (util/info "Adding cached jar %s...\n" tmpjar-path))
                      (file/hard-link cached-jar (doto (io/file ~(.getPath tmpdir) tmpjar-path)
                                                   io/make-parents))))))))
          (boot/commit! (boot/add-resource fileset tmpdir)))))))

                  ;; (recur (rest jars)
                  ;;        fs

              ;;     ;;(println "dest: " dest)
              ;;     (file/hard-link jar dest)
              ;;     #_(file/copy-atomically jar dest)))
              ;; ;;~(buber out-dir) ;; :as-jars true :exclude-scope #{"provided"})
              ;; ))))))

            ;; #_(pod/copy-dependency-jar-entries pod/env
            ;;                                    ~(.getPath out-dir)
            ;;                                    '~coords
            ;;                                    #"^.*clj$"
            ;;                                    )))))))

    ;; #_(comp
    ;;  (builtin/uber :as-jars true :exclude-scope #{"provided"})
    ;;  ;; (builtin/sift :include #{#"zip$"} :invert true)
    ;;  ;; (builtin/sift :include #{#".*appengine-api-.*jar$"} :invert true)
    ;;  (builtin/sift :move {#"(.*\.jar$)" (str mod "/" lib-dir "/$1")})
    ;;  (builtin/sift :include #{#".*jar$"}))))

;; FIXME: drive this from logging.edn
(boot/deftask logging
  "configure gae logging"
  [l log LOG kw ":log4j or :jul"
   v verbose bool "Print trace messages."
   o odir ODIR str "output dir"]
  ;; (print-task "logging" *opts*)
  (let [content (stencil/render-file
                 (if (= log :log4j)
                   "migae/boot_gae/log4j.properties.mustache"
                   "migae/boot_gae/logging.properties.mustache")
                   config-map)
        odir (if odir odir
                 (condp = log
                   :jul web-inf-dir
                   :log4j classes-dir
                   nil web-inf-dir
                   (throw (IllegalArgumentException. (str "Unrecognized :log value: " log)))))
        out-path (condp = log
                   :log4j "log4j.properties"
                   :jul "logging.properties"
                   nil  "logging.properties")
        mv-arg {(re-pattern out-path) (str odir "/$1")}]
    ;; (println "STENCIL: " content)
    ;; (println "mv pattern: " mv-arg)
    (comp
     (boot/with-pre-wrap fs
       (let [tmp-dir (boot/tmp-dir!)
             ;; _ (println "odir: " odir)
             out-file (doto (io/file tmp-dir (str odir "/" out-path)) io/make-parents)]
         (spit out-file content)
         (util/info "Configuring logging...\n")
         (-> fs (boot/add-resource tmp-dir) boot/commit!))))))

(boot/deftask monitor
  "watch etc. for gae project"
  [s service bool "service"
   v verbose bool "verbose"]
  ;;(let [mod (str (-> (boot/get-env) :gae :module :name))]
  (comp (builtin/watch)
        (builtin/notify :audible true)
        ;;(builtin/sift :move {#"(.*\.clj$)" (str (if mod (str mod "/")) classes-dir "/$1")})
        (builtin/sift :move {#"(.*\.clj$)" (str classes-dir "/$1")})
        (if service
          (target :no-clean true :service service :monitor true)
          (target :no-clean true :service service :monitor false))))

;; FIXME: not sure what this was for.  Just moving clj sources to the right output dir?
;; replaced by sifting?
#_(boot/deftask make
  "dev build, source only"
  [v verbose bool "verbose"]
  (let [mod (str (-> (boot/get-env) :gae :module))
        prev (atom nil)
        cache-dir (boot/cache-dir! :boot-gae/build)
        tmpdir (boot/tmp-dir!)]
    ;; (println "MODULE: " mod)
    (boot/empty-dir! tmpdir)
    (comp
     (boot/with-pre-wrap [fileset]
       (let [changes (->> (boot/fileset-diff @prev fileset)
                          boot/output-files
                          (map :path))]
             ;; fs (boot/new-fileset)]
         (doseq [path changes]
           (if verbose (util/info (str "changed: " path "\n")))
           (let [in-file (boot/tmp-file (boot/tmp-get fileset path))<commit_msg>make assemble task use earxml, target<commit_after>(ns migae.boot-gae
  {:boot/export-tasks true}
  (:require [clojure.java.io :as io]
            [clojure.string :as str]
            [clojure.set :as set]
            [clojure.pprint :as pp]
            [me.raynes.fs :as fs]
            [stencil.core :as stencil]
            [boot.user]
            [boot.pod :as pod]
            [boot.core :as boot]
            [boot.util :as util]
            [boot.task.built-in :as builtin]
            [boot.file            :as file]
            [boot.from.digest     :as digest])
  (:import [com.google.appengine.tools KickStart]
           [java.io File]
           [java.net URL URLClassLoader]))

            ;; [deraen.boot-less.version :refer [+version+]]))

(def boot-version "2.7.1")

(def web-xml-edn "web.xml.edn")
(def webapp-edn "webapp.edn")
(def gae-edn "appengine.edn")
(def appstats-edn "appstats.edn")

;; for microservices app:
(def services-edn "services.edn")


(def meta-inf-dir "META-INF")
(def web-inf-dir "WEB-INF")
  ;; (let [mod (-> (boot/get-env) :gae :module :name)]
  ;;   (str (if mod (str mod "/")) "WEB-INF")))

;; http://stackoverflow.com/questions/2751033/clojure-program-reading-its-own-manifest-mf
(defn manifest-map
  "Returns the mainAttributes of the manifest of the passed in class as a map."
  [j]
  (println "JAR: " j (type j))
  (->> (str "jar:file:" j "!/META-INF/MANIFEST.MF")
       clojure.java.io/input-stream
       java.util.jar.Manifest.
       .getMainAttributes
       (map (fn [[k v]] [(str k) v]))
       (into {})))

(defn expand-home [s]
  (if (or (.startsWith s "~") (.startsWith s "$HOME"))
    (str/replace-first s "~" (System/getProperty "user.home"))
    s))

(def sdk-string (let [jars (pod/resolve-dependency-jars (boot/get-env) true)
                      zip (filter #(.startsWith (.getName %) "appengine-java-sdk") jars)]
                  (if (empty? zip)
                    (println "appengine-java-sdk zipfile not found")
                    (let [fname (first (for [f zip] (.getName f)))
                          sdk-string (subs fname 0 (.lastIndexOf fname "."))]
                      ;; (println "sdk-string: " sdk-string)
                      sdk-string))))

(def config-map (merge {:build-dir "target"
                        :sdk-root (let [dir (str (System/getenv "HOME") "/.appengine-sdk")]
                                    (str dir "/" sdk-string))}
                       (reduce (fn [altered-map [k v]] (assoc altered-map k
                                                              (if (= k :sdk-root)
                                                                (str (expand-home v) "/" sdk-string)
                                                                v)))
                               {}
                               (:gae (boot/get-env)))))

;;(str (:build-dir config-map)

(def classes-dir (str web-inf-dir "/classes"))

(def lib-dir (str web-inf-dir "/lib"))

(def appcfg-class "com.google.appengine.tools.admin.AppCfg")

;; for multiple subprojects:
(def root-dir "")
(def root-project "")

(def project-dir (System/getProperty "user.dir"))
(def build-dir (str/join "/" [project-dir "target"]))

(defn output-libs-dir [nm] (str/join "/" [build-dir "libs"]))
(defn output-resources-dir [nm] (str/join "/" [build-dir "resources" nm]))

(defn java-source-dir [nm] (str/join "/" [project-dir "src" nm "java"]))
(defn input-resources-dir [nm] (str/join "/" [project-dir "src" nm "resources"]))

(defn gae-app-dir []
  (let [mod (str (-> (boot/get-env) :gae :module :name))]
    (str "target" (if mod (str "/" mod)))))

(def sdk-root-property "appengine.sdk.root")
(def java-classpath-sys-prop-key "java.class.path")
(def sdk-root-sys-prop-key "appengine.sdk.root")

(defn print-task
  [task opts]
  (if (or (:verbose opts) (:verbose config-map) (:list-tasks config-map))
    (println "TASK: " task)))

(defn dump-props []
  (println "project-dir: " project-dir)
  (println "build-dir: " build-dir)
  (println "classes-dir: " classes-dir)
  (println "output-resources-dir: " (output-resources-dir nil))
  (println "java-source-dir: " (java-source-dir nil))
  (println "input-resources-dir: " (input-resources-dir nil))
  )

(defn dump-env []
  (let [e (boot/get-env)]
    (println "ENV:")
    (util/pp* e)))


(defn dump-tmpfiles
  [lbl tfs]
  (println "\n" lbl ":")
  (doseq [tf tfs] (println (boot/tmp-file tf))))

(defn dump-tmpdirs
  [lbl tds]
  (println "\n" lbl ":")
  (doseq [td tds] (println td)))

(defn dump-fs
  [fileset]
  (doseq [f (:dirs fileset)] (println "DIR: " f))
  (doseq [f (:tree fileset)] (println "F: " f)))
  ;; (dump-tmpfiles "INPUTFILES" (boot/input-files fileset))
  ;; (dump-tmpdirs "INPUTDIRS" (boot/input-dirs fileset))
  ;; (dump-tmpdirs "OUTPUTFILESET" (boot/output-files (boot/output-fileset fileset)))
  ;; (dump-tmpfiles "OUTPUTFILES" (boot/output-files fileset))
  ;; (dump-tmpdirs "OUTPUTDIRS" (boot/output-dirs fileset))
  ;; (dump-tmpfiles "USERFILES" (boot/user-files fileset))
  ;; (dump-tmpdirs "USERDIRS" (boot/user-dirs fileset)))

(def config-props
  {;; https://docs.gradle.org/current/userguide/war_plugin.html
   :web-app-dir-name "src/main/webapp" ;; String
   ;; :web-app-dir "project-dir/web-app-dir-name" ;; File
   })

#_(def runtask-params-defaults
  {;; :http-address 127.0.0.1
   ;; :http-port 8080
   ;; :daemon false
   ;; :disable-update-check false
   ;; :disable-datagram false
   ;; :jvm-flags []
   :allow-remote-shutdown true})
   ;; :download-sdk false

(def kw->opt
  {
   :allow-remote-shutdown "--allow_remote_shutdown"
   :default-gcs-bucket "--default_gcs_bucket"
   :disable-filesapi-warning "-disable_filesapi_warning"
   :disable_restricted_check "--disable_restricted_check"
   :disable-update-check "--disable_update_check"
   :enable_filesapi "--enable_filesapi"
   :enable-jacoco "--enable_jacoco"
   :jacoco-agent-jar "--jacoco_agent_jar"
   :jacoco-agent-args "--jacoco_agent_args"
   :jacoco-exec "--jacoco_exec"
   :external-resource-dir "--external_resource_dir"
   :generated-war-dir "--generated_dir" ;; "Set the directory where generated files are created."
   :generate-war "--generate_war"
   :http-address "--address"
   :http-port "--port"
   :instance-port "--instance_port"
   :jvm-flags "--jvm_flags"
   :no-java-agent "--no_java_agent"
   :property "--property" ;; ????
   :sdk-server "--server"  ;; DevAppServer param
   :sdk-root "--sdk_root"
   :start-on-first-thread "--startOnFirstThread"})

(defn ->args [param-map]
  (let [r (flatten (for [[k v] param-map]
                      (if (= k :jvm-flags)
                        (let [flags (str/split (first v) #" ")
                              fargs (into []
                                          (for [flag flags] (str "--jvm-flag=\"" flag "\"")))]
                          (do ;(println "FLAGS: " flags (type flags) (type (first flags)))
                              ;(println "FARGS: " fargs (type fargs))
                              (seq fargs)))
                        (str (get kw->opt k) "=" v))))]
    #_(println "MERGE: " (pr-str r))
    r))

(defn- find-mainfiles [fs]
  (->> fs
       boot/input-files
       (boot/by-ext [".clj"])))


(defn get-tools-jar []
  (let [file-sep (System/getProperty "file.separator")
        ;; _ (println "File Sep: " file-sep)
        tools-api-jar (str/join file-sep [(:sdk-root config-map) "lib" "appengine-tools-api.jar"])]
    (if (not (.exists (io/as-file tools-api-jar)))
      (throw (Exception. (str "Required library 'appengine-tools-api.jar' could not be found in specified path: " tools-api-jar "!"))))
    tools-api-jar))

(defn validate-tools-api-jar []
  (let [tools-api-jar (get-tools-jar)
        path-sep (File/pathSeparator)
        jcp (System/getProperty java-classpath-sys-prop-key)]
    #_(if (not (.contains jcp tools-api-jar))
        (System/setProperty java-classpath-sys-prop-key (str/join path-sep [jcp tools-api-jar])))
    (System/setProperty java-classpath-sys-prop-key tools-api-jar)
    #_(println "Java classpath: " (System/getProperty java-classpath-sys-prop-key))

    ;; Adding appengine-tools-api.jar to context ClassLoader
    ;; see also http://stackoverflow.com/questions/194698/how-to-load-a-jar-file-at-runtime?lq=1
    (let [;; ClassLoader rootClassLoader = ClassLoader.systemClassLoader.parent
          root-class-loader (.getParent (ClassLoader/getSystemClassLoader))
          ;;URLClassLoader appengineClassloader
          ;;  = new URLClassLoader([new File(appEngineToolsApiJar).toURI().toURL()] as URL[], rootClassLoader)
          gae-class-loader (let [tools-jar-url [(.toURL (.toURI (io/as-file tools-api-jar)))]]
                                 (URLClassLoader. (into-array tools-jar-url) root-class-loader))]
      ;; Thread.currentThread().setContextClassLoader(appengineClassloader)
      (.setContextClassLoader (Thread/currentThread) gae-class-loader))))

;; (boot/deftask foo "" []
;;   (boot/with-pre-wrap [fs]
;;     (println "FS: " (type fs))
;;     ;; (let [asset-files (->> fs boot/output-files (boot/by-ext [".clj"]))]
;;     ;;      (doseq [asset asset-files] (println "foo" asset)))
;;     fs))

;; (boot/deftask clj
;;   "Assetize and mv source .clj files to <build-dir>/WEB-INF/classes"
;;   []
;;   (builtin/sift :move {#"(.*\.clj$)" (str web-inf-dir "/classes/$1")}
;;                 :to-resource #{#"clj$"}))

;; (boot/deftask aot
;;   "Built-in aot does not allow custom *compile-path*"

;;   [a all          bool   "Compile all namespaces."
;;    d dir PATH str "where to place generated class files"
;;    n namespace NS #{sym} "The set of namespaces to compile."]

;;   (let [tgt         (boot/tmp-dir!)
;;         dir         (if dir dir "")
;;         out-dir     (io/file tgt dir)
;;         foo         (doto (io/file out-dir "foo") io/make-parents)
;;         pod-env     (update-in (boot/get-env) [:directories] conj (.getPath out-dir))
;;         compile-pod (future (pod/make-pod pod-env))]
;;     (boot/with-pre-wrap [fs]
;;       (boot/empty-dir! tgt)
;;       (let [all-nses (->> fs boot/fileset-namespaces)
;;             nses     (->> all-nses (set/intersection (if all all-nses namespace)) sort)]
;;         (pod/with-eval-in @compile-pod
;;           (require '[clojure.java.io :as io])
;;           (let [foo ~(.getPath foo)]
;;             (io/make-parents (io/as-file foo))
;;             ;;(pod/add-classpath (io/as-file outdir))
;;             (binding [*compile-path* ~(.getPath out-dir)]
;;               (doseq [[idx ns] (map-indexed vector '~nses)]
;;                 (boot.util/info "Compiling %s/%s %s...\n" (inc idx) (count '~nses) ns)
;;                 (compile ns)))))
;;         (-> fs (boot/add-resource tgt) boot/commit!)))))

(boot/deftask appengine
  "generate gae appengine-web.xml"
  [c config-syms SYMS #{sym} "namespaced symbols bound to meta-config data"
   d dir DIR str "output dir"
   k keep bool str "keep intermediate .clj files"
   m module MODULE str "module dirpath"
   v verbose bool "Print trace messages."]
  (let [tmp-dir (boot/tmp-dir!)
        prev-pre (atom nil)
        dir (if dir dir web-inf-dir)]
    (boot/with-pre-wrap fileset
      (let [webapp-edn-files (->> (boot/fileset-diff @prev-pre fileset)
                                  boot/input-files
                                  (boot/by-name [webapp-edn]))
            webapp-edn-f (condp = (count webapp-edn-files)
                           0 nil
                           1 (first webapp-edn-files)
                           (throw (Exception. "only one webapp.edn file allowed")))
            webapp-edn-c (if webapp-edn-f (-> (boot/tmp-file webapp-edn-f) slurp read-string) {})
            gae-edn-fs (->> (boot/fileset-diff @prev-pre fileset)
                            boot/input-files
                            (boot/by-name [gae-edn]))]
        (if (> (count gae-edn-fs) 1) (throw (Exception. "only one web.xml.edn file allowed")))
        (if (= (count gae-edn-fs) 0) (throw (Exception. "web.xml.edn file not found")))

        (let [gae-edn-f (first gae-edn-fs)
              gae-forms (-> (boot/tmp-file gae-edn-f) slurp read-string)
              gae-forms (merge gae-forms
                               (if (-> webapp-edn-c :module)
                                 (if (-> webapp-edn-c :module :default)
                                   {}
                                   {:module (str (-> webapp-edn-c :module :name))})
                                 {}))
              app-id (name (-> (boot/get-env) :gae :app-id))
              version (str/replace (-> (boot/get-env) :gae :version) "." "-")
              version (str "r-" (subs version 0 (.lastIndexOf version "-SNAPSHOT")))
              module (-> (boot/get-env) :gae :module :name)
              ;; module (if (:module gae-forms)
              ;;          (str (:module gae-forms) "-" version)
              ;;          nil)
              gae-map (assoc gae-forms
                             :app-id app-id
                             :version version
                             :module {:name module})]
          ;; (println "gae-map: " gae-map)
          (let [content (stencil/render-file
                         "migae/boot_gae/xml.appengine-web.mustache"
                         gae-map)]
            (if verbose (println content))
            (util/info "Configuring appengine-web.xml\n")
            (let [xml-out-path (str dir "/appengine-web.xml")
                  xml-out-file (doto (io/file tmp-dir xml-out-path) io/make-parents)
                  ]
              (spit xml-out-file content)))))
      (-> fileset (boot/add-resource tmp-dir) boot/commit!))))

;; (defn- add-appstats!
;;   [reloader-ns urls in-file out-file]
;;   (let [spec (-> in-file slurp read-string)]
;;     (util/info "Adding :appstats to %s...\n" (.getName in-file))
;;     (io/make-parents out-file)
;;     (let [m (-> spec
;;                 (assoc-in [:reloader]
;;                           {:ns reloader-ns
;;                            :name "reloader"
;;                            :display {:name "Clojure reload filter"}
;;                            :urls (if (empty? urls) [{:url "./*"}]
;;                                      (merge (for [url urls] {:url url})))
;;                            :desc {:text "Clojure reload filter"}}))
;;           s (with-out-str (pp/pprint m))]
;;     (spit out-file s))))

(boot/deftask appstats
  "enable GAE Appstats"
  [k keep bool "keep intermediate .clj files"
   ;; n gen-reloader-ns NS sym "ns for gen-reloader"
   ;; r reloader-impl-ns NS sym "ns for reloader"
   w web-inf WEB-INF str "WEB-INF dir, default: WEB-INF"
   v verbose bool "Print trace messages."]
  (let [edn-tmp (boot/tmp-dir!)
        prev-pre (atom nil)
        web-inf (if web-inf web-inf web-inf-dir)

        ;; gen-reloader-ns (if gen-reloader-ns (symbol gen-reloader-ns) (gensym "reloadergen"))
        ;; gen-reloader-path (str gen-reloader-ns ".clj")
        ]
    (comp
     (boot/with-pre-wrap [fileset]
       (let [web-xml-edn-files (->> (boot/user-files fileset)
                                    (boot/by-re [(re-pattern (str web-xml-edn "$"))]))
                    ;; (boot/by-name [web-xml-edn]))
             web-xml-edn-f (condp = (count web-xml-edn-files)
                             0 (do (util/info (str "Creating " web-xml-edn "\n"))
                                   nil)
                             1 (first web-xml-edn-files)
                             (throw (Exception. "only one web.xml.edn file allowed")))
             web-xml-edn-c (if web-xml-edn-f (-> (boot/tmp-file web-xml-edn-f) slurp read-string) {})]
         ;; (println "web-xml-edn-c: " web-xml-edn-c)
           (if (:appstats web-xml-edn-c)
             fileset
             (do
               ;; (add-appstats! reloader-impl-ns urls in-file out-file)
               (let [appstats-fs (->> (boot/input-files fileset)
                                      (boot/by-name [appstats-edn]))]
                 (if (> (count appstats-fs) 1)
                   (throw (Exception. (str "only one " appstats-edn " file allowed"))))
                 (if (= (count appstats-fs) 0)
                   (throw (Exception. (str appstats-edn " file not found"))))
                 (let [appstats-edn-f (first appstats-fs)
                       appstats-config (-> (boot/tmp-file appstats-edn-f) slurp read-string)]
                   (util/info (str "Elaborating " web-xml-edn " with :appstats stanza\n"))
                   (let [m (-> web-xml-edn-c
                               (assoc-in [:appstats] (:appstats appstats-config)))
                         web-xml-edn-s (with-out-str (pp/pprint m))
                         web-xml-edn-out-file (io/file edn-tmp (boot/tmp-path web-xml-edn-f))]
                     (io/make-parents web-xml-edn-out-file)
                     (spit web-xml-edn-out-file web-xml-edn-s)))))))
               (reset! prev-pre
                       (-> fileset
                           (boot/add-source edn-tmp)
                           boot/commit!))))))

(declare earxml filters install-sdk libs logging reloader servlets target webxml)

(boot/deftask assemble
  "assemble a service-based app (ear)"
  [v verbose bool "Print trace messages."]
  (let [tmpdir (boot/tmp-dir!)
        dfl-scopes #{"compile" "runtime" "provided"}
        scopes dfl-scopes
        ;; scopes     (-> dfl-scopes
        ;;                (set/union include-scope)
        ;;                (set/difference exclude-scope))
        scope?     #(contains? scopes (:scope (util/dep-as-map %)))
        checkouts  (->> (boot/get-checkouts)
                        (filter (comp scope? :dep val))
                        (into {}))
        co-jars    (->> checkouts (map (comp :jar val)))
        co-dirs    (->> checkouts (map (comp :dir val)))

        checkout-vec (-> (boot/get-env) :checkouts)
        cos (map #(assoc (apply hash-map %) :coords [(first %) (second %)]) checkout-vec)
        ]
    (comp
     (earxml)
     (boot/with-pre-wrap [fileset]
       (doseq [co cos]
         (let [mod (if (:default co) "default" (if-let [mod (:module co)] mod "default"))]
           (let [corec (get checkouts (first (:coords co)))
                 co-dir (:dir corec)
                 jar-path (.getPath (:jar corec))
                 out-dir (doto (io/file tmpdir mod) io/make-parents)]
             ;; (println "co-dir: " co-dir)
             ;; (println "out-dir: " out-dir)
             (fs/copy-dir co-dir out-dir))))
       (-> fileset
           (boot/add-asset tmpdir :exclude #{(re-pattern (str "/META-INF/.*"))})
           (boot/commit!)))
     (target)
     )))

(boot/deftask build
  "assemble, configure, and build app"
  [k keep bool "keep intermediate .clj and .edn files"
   p prod bool "production build, without reloader"
   s service bool "build a service"
   v verbose bool "verbose"]
  (let [keep (or keep false)
        verbose (or verbose false)]
        ;; mod (str (-> (boot/get-env) :gae :module :name))]
    ;; (println "MODULE: " mod)
    (comp (install-sdk)
          (libs :verbose verbose)
          (appstats :verbose verbose)
          (builtin/javac)
          (if prod identity (reloader :keep keep :service service :verbose verbose))
          (filters :keep keep :verbose verbose)
          (servlets :keep keep :verbose verbose)
          (logging :verbose verbose)
          (webxml :verbose verbose)
          (appengine :verbose verbose)
          (builtin/sift :move {#"(.*clj$)" (str classes-dir "/$1")})
          (builtin/sift :move {#"(.*\.class$)" (str classes-dir "/$1")})
          (if service
            (comp
             (builtin/pom)
             (builtin/jar))
            identity)
          (target :service service :verbose verbose)
          #_(builtin/sift :include #{(re-pattern (str mod ".*jar"))})
          #_(builtin/install)
          #_(target :verbose verbose)
          ;; (builtin/sift :move {#"(^.*)" (str mod "/$1")})
          ;; FIXME: use cache instead?
          ;; (builtin/target :dir #{(str "target/" mod)})
          )))

(boot/deftask target
  "target, using module name"
  [C no-clean bool "Don't clean target before writing project files"
   m monitor bool "monitoring - target service assembly"
   s service bool "building as service"
   v verbose bool "verbose"]
  (let [mod      (-> (boot/get-env) :gae :module :name)
        app-dir  (-> (boot/get-env) :gae :module :app-dir)
        dir      (if monitor
                   (str app-dir "/target/" mod)
                   (if service (str "target/" mod)
                       "target"))]
    (if service
      (if (not (and mod app-dir))
        (throw (Exception. "For service targets, both :name and :app-dir must be specified in :gae map of build.boot"))))
    (println "TARGET DIR: " dir)
    (builtin/target :dir #{dir}
                    :no-clean (or no-clean false))))

(boot/deftask cache
  "control cache: retrieve, save, clean"
  [c clean bool "clean config - clear cache first"
   t trace bool "dump cache to stdout if verbose=true"
   r retrieve bool "retrieve cached master edn config file and sync to fileset (default)"
   s save bool "save master edn config file from fileset to cache"
   v verbose bool "verbose"]
  ;; default to retrieve
  (if (and retrieve save)
    (util/exit-error
     (util/fail "boot-gae/cache: only one of :retrieve and :save may be specified\n")))
  (let [retrieve (or retrieve (not save))
        save (or (not retrieve) save)]
    (boot/with-pre-wrap fileset
      (let [cache-dir (boot/cache-dir! :boot-gae/build)]
        (if verbose (println "cache-dir: " (.getPath cache-dir)))
        (if clean
          (do (if verbose (util/info (str "Clearing boot-gae cache\n")))
              (boot/empty-dir! cache-dir)
              (boot/commit! fileset))
          (if retrieve
            (do (println "RETRIEVING " (type cache-dir))
                (let [;;fs (boot/add-cached-asset fileset (io/file "boot-gae/build") identity)
                      fs (boot/add-asset fileset cache-dir)]
                  #_(println "FS: " fs)
                  (boot/commit! fs)))
            (if save
              (let [_ (println "SAVING")
                    fs (into [] (boot/output-dirs fileset))]
                ;;(println "fs: " fs (type fs))
                (apply boot/sync! cache-dir fs)
                fileset)
              fileset)))))))

(boot/deftask earxml
  "generate xml config files for microservices app"
  [d dir DIR str "output dir"
   c config-syms SYMS #{sym} "namespaced symbols bound to meta-config data"
   k keep bool str "keep intermediate .clj files"
   v verbose bool "Print trace messages."]
  (let [edn-tmp (boot/tmp-dir!)
        prev-pre (atom nil)
        odir (if dir dir meta-inf-dir)]
    ;; FIXME create services.edn map from :checkouts
    (boot/with-pre-wrap fileset
      (let [services-fs (->> (boot/fileset-diff @prev-pre fileset)
                                boot/input-files
                                (boot/by-re [(re-pattern (str services-edn "$"))]))]
            ;; (boot/by-name [services]))]
        (if (> (count services-fs) 1) (throw (Exception. "only one services.edn file allowed")))
        (if (= (count services-fs) 0) (throw (Exception. "services.edn file not found")))

        (let [services-f (first services-fs)
              services-cfg (-> (boot/tmp-file services-f) slurp read-string)
              path     (boot/tmp-path services-f)
              in-file  (boot/tmp-file services-f)
              out-file (io/file edn-tmp path)]
          (let [appengine-application-cfg (stencil/render-file
                                           "migae/boot_gae/xml.appengine-application.mustache"
                                           services-cfg)
                application-cfg (stencil/render-file
                                 "migae/boot_gae/xml.application.mustache"
                                 services-cfg)
                manifest-cfg (stencil/render-file
                              "migae/boot_gae/MANIFEST.MF.mustache"
                              services-cfg)]
            (if verbose
              (do (println appengine-application-cfg)
                  (println application-cfg)
                  (println manifest-cfg)))
            (let [appengine-application-out-path (str odir "/appengine-application.xml")
                  appengine-application-out-file (doto
                                                     (io/file edn-tmp appengine-application-out-path)
                                                   io/make-parents)
                  application-out-path (str odir "/application.xml")
                  application-out-file (doto (io/file edn-tmp application-out-path)
                                         io/make-parents)
                  manifest-out-path (str odir "/MANIFEST.MF")
                  manifest-out-file (doto (io/file edn-tmp manifest-out-path)
                                      io/make-parents)
                  ]
              (util/info "Configuring microservices app\n")
              (spit appengine-application-out-file appengine-application-cfg)
              (spit application-out-file application-cfg)
              (spit manifest-out-file manifest-cfg)))))
      (-> fileset (boot/add-resource edn-tmp) boot/commit!))))

(boot/deftask deploy
  "Installs a new version of the application onto the server, as the default version for end users."
  ;; options from AppCfg.java, see also appcfg.sh --help
  [s server SERVER str "--server"
   e email EMAIL str   "The username to use. Will prompt if omitted."
   H host  HOST  str   "Overrides the Host header sent with all RPCs."
   p proxy PROXYHOST str "'PROXYHOST[:PORT]'.  Proxies requests through the given proxy server."
   _ proxy-https PROXYHOST str "'PROXYHOST[:PORT].  Proxies HTTPS requests through the given proxy server."
   _ no-cookies bool      "Do not save/load access credentials to/from disk."
   _ sdk-root ROOT str   "Overrides where the SDK is located."
   b build-dir BUILD str "app build dir"
   _ passin   bool          "Always read the login password from stdin."
   A application APPID str "application id"
   M module MODULE str      "module"
   V version VERSION str   "(major) version"
   _ oauth2 bool            "Ignored (OAuth2 is the default)."
   _ noisy bool             "Log much more information about what the tool is doing."
   _ enable-jar-splitting bool "Split large jar files (> 10M) into smaller fragments."
   _ jar-splitting-excludes SUFFIXES str "list of files to be excluded from all jars"
   _ disable-jar-jsps bool "Do not jar the classes generated from JSPs."
   _ enable-jar-classes bool "Jar the WEB-INF/classes content."
   _ delete-jsps bool "Delete the JSP source files after compilation."
   _ retain-upload-dir bool "Do not delete temporary (staging) directory used in uploading."
   _ compile-encoding ENC str "The character encoding to use when compiling JSPs."
   _ num-days NUM_DAYS int "number of days worth of log data to get"
   _ severity SEVERITY int "Severity of app-level log messages to get"
   _ include-all bool   "Include everything in log messages."
   a append bool          "Append to existing file."
   _ num-runs NUM-RUNS int "Number of scheduled execution times to compute."
   f force bool           "Force deletion of indexes without being prompted."
   _ no-usage-reporting bool "Disable usage reporting."
   D auto-update-dispatch bool "Include dispatch.yaml in updates"
   _ sdk-help bool "Display SDK help screen"
   v verbose bool          "Print invocation args"]
  (if verbose (println "TASK: boot-gae/deploy"))
  (validate-tools-api-jar)
  (println "PARAMS: " *opts*)
  (let [opts (merge {:sdk-root (:sdk-root config-map)
                     ;; :use-java7 true
                     :build-dir (gae-app-dir)} ;; (:build-dir config-map)}
                    *opts*)
        _ (println "OPTS: " opts)
        params (into [] (for [[k v] (remove (comp nil? second)
                                            (dissoc opts :build-dir :verbose :sdk-help))]
                          (str "--" (str/replace (name k) #"-" "_")
                               (if (not (instance? Boolean v)) (str "=" v)))))
        params (if (:sdk-help *opts*)
                 ["help" "update"]
                 (conj params "update" (:build-dir opts)))
        params (into-array String params)
        ;; ClassLoader classLoader = Thread.currentThread().contextClassLoader
        class-loader (-> (Thread/currentThread) (.getContextClassLoader))
        cl (.getParent class-loader)
        app-cfg (Class/forName appcfg-class true class-loader)]

  ;; def appCfg = classLoader.loadClass(APPENGINE_TOOLS_MAIN)
  ;; appCfg.main(params as String[])
    (def method (first (filter #(= (. % getName) "main") (. app-cfg getMethods))))
    (def invoke-args (into-array Object [params]))
    (if verbose ;; (or (:verbose *opts*) (:verbose config-map))
      (do (println "CMD: AppCfg")
          (doseq [arg params]
            (println "\t" arg))))
    (. method invoke nil invoke-args)))

(boot/deftask filters
  "generate filters; update web.xml.edn with filter config data"

  [k keep bool "keep intermediate .clj files"
   n gen-filters-ns NS str "namespace to generate and aot; default: 'filters"
   w web-inf WEB-INF str "WEB-INF dir, default: WEB-INF"
   v verbose bool "Print trace messages."]
  (let [edn-tmp-dir (boot/tmp-dir!)
        prev-pre (atom nil)
        ;; config-sym (if config-sym config-sym 'filters/config)
        ;; config-ns (symbol (namespace config-sym))
        web-inf (if web-inf web-inf web-inf-dir)
        filters-edn "filters.edn"
        gen-filters-tmp-dir (boot/tmp-dir!)
        gen-filters-ns (if gen-filters-ns (symbol gen-filters-ns) (gensym "filtersgen"))
        gen-filters-path (str gen-filters-ns ".clj")]
    (comp
     (boot/with-pre-wrap [fileset]
       (let [web-xml-edn-files (->> (boot/fileset-diff @prev-pre fileset)
                                    boot/input-files
                                    (boot/by-re [(re-pattern (str web-xml-edn "$"))]))]
                    ;; (boot/by-name [web-xml-edn]))]
         (if (> (count web-xml-edn-files) 1)
           (throw (Exception. "only one web.xml.edn file allowed")))
         (if (= (count web-xml-edn-files) 0)
           (throw (Exception. "cannot find web.xml.edn")))

         (let [web-xml-edn-f (first web-xml-edn-files)
               web-xml-edn-c (-> (boot/tmp-file web-xml-edn-f) slurp read-string)]
           (if (:filters web-xml-edn-c)
             fileset
             (do
               ;; step 0: read the edn files
               (let [filters-edn-files (->> (boot/input-files fileset)
                                            (boot/by-name [filters-edn]))]
                 (if (> (count filters-edn-files) 1)
                   (throw (Exception. "only one filters.edn file allowed")))
                 (if (= (count filters-edn-files) 0)
                   (throw (Exception. "cannot find filters.edn")))
                 (let [edn-filters-f (first filters-edn-files)
                       filter-configs (-> (boot/tmp-file edn-filters-f) slurp read-string)
                       smap (-> web-xml-edn-c (assoc-in [:filters]
                                                        (:filters filter-configs)))
                       web-xml-edn-s (with-out-str (pp/pprint smap))]
                   ;; step 1:  inject filter config stanza to web.xml.edn
                   (let [path     (boot/tmp-path web-xml-edn-f)
                         web-xml-edn-in-file  (boot/tmp-file web-xml-edn-f)
                         web-xml-edn-out-file (io/file edn-tmp-dir path)]
                     (io/make-parents web-xml-edn-out-file)
                     (spit web-xml-edn-out-file web-xml-edn-s))

                   ;; step 2: gen filters
                   (let [gen-filters-content (stencil/render-file "migae/boot_gae/gen-filters.mustache"
                                                                   (assoc filter-configs
                                                                          :gen-filters-ns
                                                                          gen-filters-ns))
                         gen-filters-out-file (doto
                                                   (io/file gen-filters-tmp-dir gen-filters-path)
                                                 io/make-parents)]
                     (spit gen-filters-out-file gen-filters-content))))))))
       (util/info "Configuring filters...\n")

       ;; step 3: commit files to fileset
       (reset! prev-pre
               (-> fileset
                   (boot/add-source edn-tmp-dir)
                   (boot/add-source gen-filters-tmp-dir)
                   boot/commit!)))
     (builtin/aot :namespace #{gen-filters-ns}) ;; :dir (str web-inf "/classes"))
     (if keep identity
       (builtin/sift :include #{(re-pattern (str gen-filters-ns ".*.class"))}
                     :invert true))
     (if keep
       (comp
        (builtin/sift :to-resource #{(re-pattern web-xml-edn)})
        (builtin/sift :to-asset #{(re-pattern gen-filters-path)}))
       identity)
     )))

(boot/deftask install-sdk
  "Unpack and install the SDK zipfile"
  [r release SDKREL str "SDK verion"
   v verbose bool "Print trace messages"]
  ;;NB: java property expected by kickstart is "appengine.sdk.root"
  ;; (print-task "install-sdk" *opts*)
  (let [release (or release "RELEASE")
        coords (vector 'com.google.appengine/appengine-java-sdk
                       "RELEASE"
                       :extension "zip")
        jar-path (pod/resolve-dependency-jar (boot/get-env) coords)
        sdk-dir (io/as-file (:sdk-root config-map))
        prev        (atom nil)]
    (boot/with-pre-wrap fileset
      (if (.exists sdk-dir)
        (do
          (let [file-sep (System/getProperty "file.separator")
                tools-api-jar (str/join file-sep [(:sdk-root config-map) "lib" "appengine-tools-api.jar"])]
            (if (not (.exists (io/as-file tools-api-jar)))
              (do
                (println (str "Found sdk-dir " sdk-dir " but not its contents; re-exploding"))
                (boot/empty-dir! sdk-dir)
                (println "Exploding SDK\n from: " jar-path "\n to: " (.getPath sdk-dir))
                (pod/unpack-jar jar-path (.getParent sdk-dir)))
              (if (or (:verbose *opts*) (:verbose config-map))
                (println "SDK already installed at: " (.getPath sdk-dir))))))
        (do
          (if (or (:verbose *opts*) (:verbose config-map))
            (println "Installing unpacked SDK to: " (.getPath sdk-dir)))
          (pod/unpack-jar jar-path (.getParent sdk-dir))))
      fileset)))

;;(core/deftask buber
;; FIXME: what was this for???
#_(defn buber
  [tgt]
  (println "BUBER ")
  (let [;; tgt        (boot/tmp-dir!)
        cache      (boot/cache-dir! ::uber :global true)
        dfl-scopes #{"compile" "runtime"}
        scopes     dfl-scopes
        ;; scopes     (-> dfl-scopes
        ;;                (set/union include-scope)
        ;;                (set/difference exclude-scope))
        _ (println "SCOPES: " dfl-scopes)
        scope?     #(contains? scopes (:scope (util/dep-as-map %)))
        jars       (-> pod/env ;; (boot/get-env)
                       (update-in [:dependencies] (partial filter scope?))
                       pod/resolve-dependency-jars)
        _ (println "JARS: " jars)
        jars       (remove #(.endsWith (.getName %) ".pom") jars)
        checkouts  (->> (boot/get-checkouts)
                        (filter (comp scope? :dep val))
                        (into {}))
        co-jars    (->> checkouts (map (comp :jar val)))
        ;; co-dirs    (->> checkouts (map (comp :dir val)))
        ;; exclude    (or exclude pod/standard-jar-exclusions)
        ;; merge      (or merge pod/standard-jar-mergers)
        ;; reducer    (fn [xs jar]
        ;;              (boot/add-cached-resource
        ;;                xs (digest/md5 jar) (partial pod/unpack-jar jar)
        ;;                :include include :exclude exclude :mergers merge))
        ;; co-reducer #(boot/add-resource
        ;;               %1 %2 :include include :exclude exclude :mergers merge)
        ]
    ;; (boot/with-pre-wrap [fs]
    ;; (println "co jars: " co-jars)
      (when (seq jars)
        (doseq [jar jars] ;; (reduce into [] [jars co-jars])]
          (let [dest (doto (io/file tgt (.getName jar)) io/make-parents)]
            (println "dest: " dest)
            (file/copy-atomically jar dest))))))

      ;;       (when-not (.exists src)
      ;;         (println "CACHING " name)
      ;;         (util/dbug* "Caching jar %s...\n" name)
      ;;         (file/copy-atomically jar src))
      ;;       (util/dbug* "Adding cached jar %s...\n" name)
      ;;       (println (format "Adding cached jar %s...\n" name))
      ;;       (file/hard-link src (io/file tgt name)))))
      ;; #_(boot/commit! (if as-jars
      ;;                 (boot/add-resource fs tgt)
      ;;                 (reduce co-reducer (reduce reducer fs jars) co-dirs)))))

;; FIXME:  for dev phase, we want to include deps in test scope
(boot/deftask libs
  ""
  [v verbose bool "Print trace messages."]
  (let [checkout-vec (-> (boot/get-env) :checkouts)
        cos (map #(assoc (apply hash-map %) :coords [(first %) (second %)]) checkout-vec)

        dfl-scopes #{"compile" "runtime" "provided"}
        scopes dfl-scopes
        ;; scopes     (-> dfl-scopes
        ;;                (set/union include-scope)
        ;;                (set/difference exclude-scope))
        scope?     #(contains? scopes (:scope (util/dep-as-map %)))
        checkouts  (->> (boot/get-checkouts)
                        (filter (comp scope? :dep val))
                        (into {}))
        cache      (boot/cache-dir! ::uber :global true)
        ]
    (if (empty? cos)
      (do
        ;;(println "NO CHECKOUTS")
          (comp
           (builtin/uber :as-jars true :exclude-scope #{"provided"})
           ;; (builtin/sift :include #{#"zip$"} :invert true)
           ;; (builtin/sift :include #{#".*appengine-api-.*jar$"} :invert true)
           ;; (builtin/sift :include #{#".*jar$"})
           (builtin/sift :move {#"(.*\.jar$)" (str lib-dir "/$1")})))
      (boot/with-pre-wrap [fileset]
        ;;(println "CHECKOUTS")
        (let [tmpdir     (boot/tmp-dir!)]
          (doseq [co cos]
            (let [mod (if (:default co) "default"
                          (if-let [mod (:module co)]
                            mod "default"))
                  coords (:coords co)
                  pod-env (update-in (dissoc (boot/get-env) :checkouts)
                                     [:dependencies] #(identity %2)
                                     (concat '[[boot/core "2.7.1"] ;; ~(str boot-version)]
                                               [boot/pod "2.7.1"]] ;; ~(str boot-version)]]
                                             (vector coords)))
                  pod (future (pod/make-pod pod-env))]
              (if verbose (println "MODULE: " mod))
              (pod/with-eval-in @pod
                (require '[boot.pod :as pod] '[boot.util :as util]
                         '[boot.core :as boot] '[boot.file :as file]
                         '[boot.task.built-in :as builtin] '[boot.from.digest :as digest]
                         '[clojure.java.io :as io])
                (let [dfl-scopes #{"compile" "runtime" "test"}
                      scopes dfl-scopes
                      scope? #(contains? scopes (:scope (util/dep-as-map %)))
                      co-jar (pod/resolve-dependency-jar (boot/get-env) '~coords)
                      jars   (remove #(= (str co-jar) (.getPath %))
                                     (-> pod/env
                                         (update-in [:dependencies] (partial filter scope?))
                                         (pod/resolve-dependency-jars)))]
                  (doseq [jar jars]
                    (let [hash (digest/md5 jar)
                          name (str hash "-" (.getName jar))
                          tmpjar-path (str ~mod "/WEB-INF/lib/" (.getName jar))
                          cached-jar  (io/file ~(.getPath cache) hash)]
                      (when-not (.exists cached-jar)
                        (if ~verbose (util/info "Caching jar %s...\n" name))
                        (file/copy-atomically jar cached-jar))
                      (if ~verbose (util/info "Adding cached jar %s...\n" tmpjar-path))
                      (file/hard-link cached-jar (doto (io/file ~(.getPath tmpdir) tmpjar-path)
                                                   io/make-parents))))))))
          (boot/commit! (boot/add-resource fileset tmpdir)))))))

                  ;; (recur (rest jars)
                  ;;        fs

              ;;     ;;(println "dest: " dest)
              ;;     (file/hard-link jar dest)
              ;;     #_(file/copy-atomically jar dest)))
              ;; ;;~(buber out-dir) ;; :as-jars true :exclude-scope #{"provided"})
              ;; ))))))

            ;; #_(pod/copy-dependency-jar-entries pod/env
            ;;                                    ~(.getPath out-dir)
            ;;                                    '~coords
            ;;                                    #"^.*clj$"
            ;;                                    )))))))

    ;; #_(comp
    ;;  (builtin/uber :as-jars true :exclude-scope #{"provided"})
    ;;  ;; (builtin/sift :include #{#"zip$"} :invert true)
    ;;  ;; (builtin/sift :include #{#".*appengine-api-.*jar$"} :invert true)
    ;;  (builtin/sift :move {#"(.*\.jar$)" (str mod "/" lib-dir "/$1")})
    ;;  (builtin/sift :include #{#".*jar$"}))))

;; FIXME: drive this from logging.edn
(boot/deftask logging
  "configure gae logging"
  [l log LOG kw ":log4j or :jul"
   v verbose bool "Print trace messages."
   o odir ODIR str "output dir"]
  ;; (print-task "logging" *opts*)
  (let [content (stencil/render-file
                 (if (= log :log4j)
                   "migae/boot_gae/log4j.properties.mustache"
                   "migae/boot_gae/logging.properties.mustache")
                   config-map)
        odir (if odir odir
                 (condp = log
                   :jul web-inf-dir
                   :log4j classes-dir
                   nil web-inf-dir
                   (throw (IllegalArgumentException. (str "Unrecognized :log value: " log)))))
        out-path (condp = log
                   :log4j "log4j.properties"
                   :jul "logging.properties"
                   nil  "logging.properties")
        mv-arg {(re-pattern out-path) (str odir "/$1")}]
    ;; (println "STENCIL: " content)
    ;; (println "mv pattern: " mv-arg)
    (comp
     (boot/with-pre-wrap fs
       (let [tmp-dir (boot/tmp-dir!)
             ;; _ (println "odir: " odir)
             out-file (doto (io/file tmp-dir (str odir "/" out-path)) io/make-parents)]
         (spit out-file content)
         (util/info "Configuring logging...\n")
         (-> fs (boot/add-resource tmp-dir) boot/commit!))))))

(boot/deftask monitor
  "watch etc. for gae project"
  [s service bool "service"
   v verbose bool "verbose"]
  ;;(let [mod (str (-> (boot/get-env) :gae :module :name))]
  (comp (builtin/watch)
        (builtin/notify :audible true)
        (builtin/sift :move {#"(.*\.clj$)" (str classes-dir "/$1")})
        (if service
          (target :no-clean true :service service :monitor true)
          (target :no-clean true :service service :monitor false))))

;; FIXME: not sure what this was for.  Just moving clj sources to the right output dir?
;; replaced by sifting?
#_(boot/deftask make
  "dev build, source only"
  [v verbose bool "verbose"]
  (let [mod (str (-> (boot/get-env) :gae :module))
        prev (atom nil)
        cache-dir (boot/cache-dir! :boot-gae/build)
        tmpdir (boot/tmp-dir!)]
    ;; (println "MODULE: " mod)
    (boot/empty-dir! tmpdir)
    (comp
     (boot/with-pre-wrap [fileset]
       (let [changes (->> (boot/fileset-diff @prev fileset)
                          boot/output-files
                          (map :path))]
             ;; fs (boot/new-fileset)]
         (doseq [path changes]
           (if verbose (util/info (str "changed: " path "\n")))
           (let [in-file (boot/tmp-file (boot/tmp-get fileset path))
{'fraction_non_alphanumeric': 0.14910063253451974, 'fraction_numerical': 0.0017414360668262046, 'mean_word_length': 1.7232063637754322, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 181, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 135}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 164, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1035919', 'n_tokens_mistral': 28545, 'n_tokens_neox': 27205, 'n_words': 7062}