<commit_before>(ns leiningen.cljs-lambda.aws
  (:require [leiningen.cljs-lambda.logging :as logging :refer [log]]
            [leiningen.cljs-lambda.args :as args]
            [cheshire.core :as json]
            [clojure.java.io :as io]
            [clojure.java.shell :as shell]
            [clojure.set :as set]
            [clojure.string :as str]
            [base64-clj.core :as base64]
            [clojure.pprint :as pprint]
            [camel-snake-kebab.core :as csk]
            [clojure.string :as string])
  (:import [java.io File]
           [java.util.concurrent Executors]))

(defn abs-path [^File f] (.getAbsolutePath f))

(defmacro with-meta-config [config & body]
  `(let [{aws-profile# :aws-profile region# :region} ~config]
     (binding [args/*aws-profile* (or aws-profile# args/*aws-profile*)
               args/*region*      (or region#      args/*region*)]
       ~@body)))

(defn meta-config []
  (cond-> {}
    args/*region*      (assoc :region  (name args/*region*))
    args/*aws-profile* (assoc :profile (name args/*aws-profile*))))

(defn update-vpc-value [{:keys [vpc] :as fn-spec}]
  (if-not vpc
    fn-spec
    (let [subnets (:subnets vpc)
          security-groups (:security-groups vpc)]
      (merge fn-spec {:vpc (string/join
                             ["SubnetIds=["
                              (string/join "," subnets)
                              "],SecurityGroupIds=["
                              (string/join "," security-groups)
                              "]"])}))))

(defn ->cli-args [m & [positional {:keys [preserve-names?]}]]
  (let [m    (cond-> (merge (meta-config) (update-vpc-value m))
               (not preserve-names?)
               (set/rename-keys {:name :function-name :vpc :vpc-config}))
        args (flatten
              (for [[k v] m]
                [(str "--" (name k))
                 (if (keyword? v) (name v) (str v))]))]
    (cond->> args positional (into positional))))

(defn aws-cli! [service cmd args & [{:keys [fatal?] :or {fatal? true}}]]
  (apply log :verbose "aws" service (name cmd) args)
  (let [{:keys [exit err] :as r}
        (apply shell/sh "aws" service (name cmd) args)]
    (if (and fatal? (not (zero? exit)))
      (leiningen.core.main/abort err)
      r)))

(def lambda-cli! (partial aws-cli! "lambda"))

(def fn-config-args
  #{:name :role :handler :description :timeout :memory-size :runtime :vpc})

(def create-function-args
  (into fn-config-args
    #{:zip-file :output :query}))

(def update-function-code-args
  (remove #{:vpc} create-function-args))

(defn validate-fn-spec! [{fn-name :name vpc :vpc}]
  (let [subnets (:subnets vpc)
        security-groups (:security-groups vpc)
        vpc-set (or (seq subnets) (seq security-groups))]
    (when vpc-set
      (when-not (and (> (count subnets) 0) (> (count security-groups) 0))
        (leiningen.core.main/abort
          "Invalid VPC spec for" fn-name "function. At least one subnet and one security group must be specified"))
      (when-not (every? string? subnets)
        (leiningen.core.main/abort
          "Invalid VPC spec for" fn-name "function. Subnets not a list of strings:" subnets))
      (when-not (every? string? security-groups)
        (leiningen.core.main/abort
          "Invalid VPC spec for" fn-name "function. Security groups not a list of strings:" security-groups)))))

(defn fn-spec->cli-args [fn-args {:keys [publish] :as fn-spec}]
  (let [args (merge {:output "text" :query "Version"} fn-spec)]
    (-> args
        (select-keys fn-args)
        (->cli-args (cond-> [] publish (conj "--publish"))))))

(defn update-alias! [alias-name fn-name version]
  (lambda-cli!
    :update-alias
    (->cli-args
      {:function-name fn-name
       :name alias-name
       :function-version version}
      nil
      {:preserve-names? true})))

(defn create-alias! [alias-name fn-name version]
  (lambda-cli!
   :create-alias
   (->cli-args
    {:function-name fn-name
     :name alias-name
     :function-version version}
    nil
    {:preserve-names? true})
   {:fatal? false}))

(def default-runtime "nodejs4.3")

(defn- create-function! [fn-spec zip-path]
  (let [args (fn-spec->cli-args
               create-function-args
               (merge {:runtime default-runtime} fn-spec {:zip-file zip-path}))]
    (-> (lambda-cli! :create-function args) :out str/trim)))

(defn- update-function-config! [fn-spec]
  (lambda-cli!
   :update-function-configuration
   (-> (merge {:vpc {:subnets [] :security-groups []}} fn-spec)
       (select-keys fn-config-args)
       ->cli-args)))

(defn- update-function-code!
  [{:keys [name publish] :as fn-spec} zip-path]
  (let [args (merge (fn-spec->cli-args
                      update-function-code-args
                      {:name name :zip-file zip-path :publish publish}))]
    (-> (lambda-cli! :update-function-code args)
        :out
        str/trim)))

(defn- get-function-configuration! [{:keys [name]}]
  (let [{:keys [exit out]} (lambda-cli!
                            :get-function-configuration
                            (->cli-args {:name name})
                            {:fatal? false})]
    (case exit
      255 nil
      0   (json/parse-string out csk/->kebab-case-keyword))))

(defn normalize-config [config]
  (-> config
      (update-in [:vpc :subnets] sort)
      (update-in [:vpc :security-groups] sort)))

(defn remote-config->local-config [remote]
  (let [remote (set/rename-keys remote {:function-name :name :vpc-config :vpc})]
    (if-let [vpc (:vpc remote)]
      (assoc remote :vpc
        (-> vpc
            (select-keys #{:subnet-ids :security-group-ids})
            (set/rename-keys {:subnet-ids :subnets :security-group-ids :security-groups})))
      remote)))

(defn same-config? [remote local]
  (let [remote (-> remote remote-config->local-config normalize-config)
        local  (-> local (select-keys fn-config-args) normalize-config)]
    (= (select-keys remote (keys local)) local)))

(defn- deploy-function!
  [zip-path {fn-name :name create? :create :as fn-spec}]
  (validate-fn-spec! fn-spec)
  (if-let [remote-config (get-function-configuration! fn-spec)]
    (do
      (when-not (same-config? remote-config fn-spec)
        (update-function-config! fn-spec))
      (update-function-code! fn-spec zip-path))
    (if create?
      (create-function! fn-spec zip-path)
      (leiningen.core.main/abort
       "Function" fn-name "doesn't exist & :create not set"))))<commit_msg>added :dead-letter configuration<commit_after>(ns leiningen.cljs-lambda.aws
  (:require [leiningen.cljs-lambda.logging :as logging :refer [log]]
            [leiningen.cljs-lambda.args :as args]
            [cheshire.core :as json]
            [clojure.java.io :as io]
            [clojure.java.shell :as shell]
            [clojure.set :as set]
            [clojure.string :as str]
            [base64-clj.core :as base64]
            [clojure.pprint :as pprint]
            [camel-snake-kebab.core :as csk]
            [clojure.string :as string])
  (:import [java.io File]
           [java.util.concurrent Executors]))

(defn abs-path [^File f] (.getAbsolutePath f))

(defmacro with-meta-config [config & body]
  `(let [{aws-profile# :aws-profile region# :region} ~config]
     (binding [args/*aws-profile* (or aws-profile# args/*aws-profile*)
               args/*region*      (or region#      args/*region*)]
       ~@body)))

(defn meta-config []
  (cond-> {}
    args/*region*      (assoc :region  (name args/*region*))
    args/*aws-profile* (assoc :profile (name args/*aws-profile*))))

(defn update-dead-letter-value [{:keys [dead-letter] :as fn-spec}]
  (if-not dead-letter
    fn-spec
    (merge fn-spec {:dead-letter (str "TargetArn=" dead-letter)})))

(defn update-vpc-value [{:keys [vpc] :as fn-spec}]
  (if-not vpc
    fn-spec
    (let [subnets (:subnets vpc)
          security-groups (:security-groups vpc)]
      (merge fn-spec {:vpc (string/join
                             ["SubnetIds=["
                              (string/join "," subnets)
                              "],SecurityGroupIds=["
                              (string/join "," security-groups)
                              "]"])}))))

(defn ->cli-args [m & [positional {:keys [preserve-names?]}]]
  (let [m    (cond-> (merge (meta-config) (-> m (update-vpc-value) (update-dead-letter-value)))
               (not preserve-names?)
               (set/rename-keys {:name :function-name :vpc :vpc-config :dead-letter :dead-letter-config}))
        args (flatten
              (for [[k v] m]
                [(str "--" (name k))
                 (if (keyword? v) (name v) (str v))]))]
    (cond->> args positional (into positional))))

(defn aws-cli! [service cmd args & [{:keys [fatal?] :or {fatal? true}}]]
  (apply log :verbose "aws" service (name cmd) args)
  (let [{:keys [exit err] :as r}
        (apply shell/sh "aws" service (name cmd) args)]
    (if (and fatal? (not (zero? exit)))
      (leiningen.core.main/abort err)
      r)))

(def lambda-cli! (partial aws-cli! "lambda"))

(def fn-config-args
  #{:name :role :handler :description :timeout :memory-size :runtime :vpc :dead-letter})

(def fn-spec-defaults
  {:vpc {:subnets [] :security-groups []} :dead-letter ""})

(def create-function-args
  (into fn-config-args
    #{:zip-file :output :query}))

(def update-function-code-args
  (remove #{:vpc :dead-letter} create-function-args))

(defn validate-fn-spec! [{fn-name :name vpc :vpc}]
  (let [subnets (:subnets vpc)
        security-groups (:security-groups vpc)
        vpc-set (or (seq subnets) (seq security-groups))]
    (when vpc-set
      (when-not (and (> (count subnets) 0) (> (count security-groups) 0))
        (leiningen.core.main/abort
          "Invalid VPC spec for" fn-name "function. At least one subnet and one security group must be specified"))
      (when-not (every? string? subnets)
        (leiningen.core.main/abort
          "Invalid VPC spec for" fn-name "function. Subnets not a list of strings:" subnets))
      (when-not (every? string? security-groups)
        (leiningen.core.main/abort
          "Invalid VPC spec for" fn-name "function. Security groups not a list of strings:" security-groups)))))

(defn fn-spec->cli-args [fn-args {:keys [publish] :as fn-spec}]
  (let [args (merge {:output "text" :query "Version"} fn-spec)]
    (-> args
        (select-keys fn-args)
        (->cli-args (cond-> [] publish (conj "--publish"))))))

(defn update-alias! [alias-name fn-name version]
  (lambda-cli!
    :update-alias
    (->cli-args
      {:function-name fn-name
       :name alias-name
       :function-version version}
      nil
      {:preserve-names? true})))

(defn create-alias! [alias-name fn-name version]
  (lambda-cli!
   :create-alias
   (->cli-args
    {:function-name fn-name
     :name alias-name
     :function-version version}
    nil
    {:preserve-names? true})
   {:fatal? false}))

(def default-runtime "nodejs4.3")

(defn- create-function! [fn-spec zip-path]
  (let [args (fn-spec->cli-args
               create-function-args
               (merge {:runtime default-runtime} fn-spec {:zip-file zip-path}))]
    (-> (lambda-cli! :create-function args) :out str/trim)))

(defn- update-function-config! [fn-spec]
  (lambda-cli!
   :update-function-configuration
   (-> (merge fn-spec-defaults fn-spec)
       (select-keys fn-config-args)
       ->cli-args)))

(defn- update-function-code!
  [{:keys [name publish] :as fn-spec} zip-path]
  (let [args (merge (fn-spec->cli-args
                      update-function-code-args
                      {:name name :zip-file zip-path :publish publish}))]
    (-> (lambda-cli! :update-function-code args)
        :out
        str/trim)))

(defn- get-function-configuration! [{:keys [name]}]
  (let [{:keys [exit out]} (lambda-cli!
                            :get-function-configuration
                            (->cli-args {:name name})
                            {:fatal? false})]
    (case exit
      255 nil
      0   (json/parse-string out csk/->kebab-case-keyword))))

(defn normalize-config [config]
  (-> config
      (update-in [:vpc :subnets] sort)
      (update-in [:vpc :security-groups] sort)))

(defn remote-config->local-config [remote]
  (let [remote (set/rename-keys remote {:function-name :name :vpc-config :vpc :dead-letter-config :dead-letter})]
    (merge
      remote
      (if-let [vpc (:vpc remote)]
        (assoc remote :vpc
          (-> vpc
              (select-keys #{:subnet-ids :security-group-ids})
              (set/rename-keys {:subnet-ids :subnets :security-group-ids :security-groups}))))
      {:dead-letter (get-in remote [:dead-letter :target-arn] "")})))

(defn same-config? [remote local]
  (let [remote (-> remote remote-config->local-config normalize-config)
        local  (-> (merge fn-spec-defaults local) (select-keys fn-config-args) normalize-config)]
    (= (select-keys remote (keys local)) local)))

(defn- deploy-function!
  [zip-path {fn-name :name create? :create :as fn-spec}]
  (validate-fn-spec! fn-spec)
  (if-let [remote-config (get-function-configuration! fn-spec)]
    (do
      (when-not (same-config? remote-config fn-spec)
        (update-function-config! fn-spec))
      (update-function-code! fn-spec zip-path))
    (if create?
      (create-function! fn-spec zip-path)
      (leiningen.core.main/abort
       "Function" fn-name "doesn't exist & :create not set"))))
{'fraction_non_alphanumeric': 0.16522193991929457, 'fraction_numerical': 0.0017934538932894933, 'mean_word_length': 2.375283732660782, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3299757', 'n_tokens_mistral': 4515, 'n_tokens_neox': 4413, 'n_words': 1111}