<commit_before>;;
;; Copyright (c) Two Sigma Open Source, LLC
;;
;; Licensed under the Apache License, Version 2.0 (the "License");
;; you may not use this file except in compliance with the License.
;; You may obtain a copy of the License at
;;
;;  http://www.apache.org/licenses/LICENSE-2.0
;;
;; Unless required by applicable law or agreed to in writing, software
;; distributed under the License is distributed on an "AS IS" BASIS,
;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
;; See the License for the specific language governing permissions and
;; limitations under the License.
;;
(ns waiter.token-request-test
  (:require [clj-time.core :as t]
            [clojure.core.async :as async]
            [clojure.data.json :as json]
            [clojure.string :as str]
            [clojure.test :refer :all]
            [clojure.tools.logging :as log]
            [clojure.walk :as walk]
            [plumbing.core :as pc]
            [waiter.service-description :as sd]
            [waiter.util.client-tools :refer :all]
            [waiter.util.date-utils :as du])
  (:import (java.net URL)
           (org.joda.time DateTime)))

(defn- waiter-url->token-url
  [waiter-url]
  (str HTTP-SCHEME waiter-url "/token"))

(defn- retrieve-token-root
  [waiter-url]
  (-> (waiter-settings waiter-url)
      (get-in [:cluster-config :name])))

(deftest ^:parallel ^:integration-fast test-update-token-cache-consistency
  (testing-using-waiter-url
    (let [service-id-prefix (rand-name "testhostname")
          token (rand-name)
          update-token-fn (fn [version]
                            (let [{:keys [status]} (post-token waiter-url {:name service-id-prefix
                                                                           :cpus 1
                                                                           :mem 1250
                                                                           :version version
                                                                           :cmd "test command"
                                                                           :health-check-url "/ping"
                                                                           :permitted-user "*"
                                                                           :run-as-user (retrieve-username)
                                                                           :token token})]
                              (is (= 200 status))))
          validate-token-fn (fn [version num-threads num-iter]
                              (parallelize-requests
                                num-threads
                                num-iter
                                (fn []
                                  (let [{:keys [body status]} (get-token waiter-url token)]
                                    (is (= 200 status))
                                    (is (every? #(str/includes? (str body) (str %)) [service-id-prefix "1250" version "test command"]))
                                    (is (not-any? #(str/includes? (str body) (str %)) ["invalid"]))))))
          update-and-validate-token-fn (fn [version]
                                         (log/info "creating configuration using token" token "with version" version)
                                         (update-token-fn version)
                                         (log/info "asserting configuration for token" token "from routers (best-effort)")
                                         (validate-token-fn version 10 10))
          version1 "123987132937213712"
          version2 "656760465406467480"
          version3 "678219671796032121"]
      (try
        (update-and-validate-token-fn version1)
        (update-and-validate-token-fn version2)
        (update-and-validate-token-fn version3)
        (finally
          (delete-token-and-assert waiter-url token))))))

(defn- name-from-service-description [waiter-url service-id]
  (get-in (service-settings waiter-url service-id) [:service-description :name]))

(defn- service-id->source-tokens-entries [waiter-url service-id]
  (some-> (service-settings waiter-url service-id)
          :source-tokens
          walk/stringify-keys
          set))

(defn- token->etag [waiter-url token]
  (-> (get-token waiter-url token :query-params {"token" token})
      :headers
      (get "etag")))

(defn- make-source-tokens-entries [waiter-url & tokens]
  (mapv (fn [token] {"token" token "version" (token->etag waiter-url token)}) tokens))

(defn- create-token-name
  [waiter-url service-id-prefix]
  (str service-id-prefix "." (subs waiter-url 0 (str/index-of waiter-url ":"))))

(defn- list-tokens
  [waiter-url owner cookies query-params]
  (let [tokens-response (make-request waiter-url "/tokens" :cookies cookies :query-params query-params)]
    (log/debug "retrieved tokens for owner " owner ":" (:body tokens-response))
    tokens-response))

(defn- parse-token-description
  "Parses a response as json and keywordizes the map."
  [response-body]
  (try
    (-> response-body str json/read-str pc/keywordize-map)
    (catch Exception _
      (is false (str "Failed to parse token " response-body)))))

(defmacro assert-token-response
  "Asserts the token data in the response"
  ([response token-root service-id-prefix deleted]
   `(assert-token-response ~response ~token-root ~service-id-prefix ~deleted true))
  ([response token-root service-id-prefix deleted include-metadata]
   `(let [body# (:body ~response)
          token-description# (parse-token-description body#)]
      (assert-response-status ~response 200)
      (when ~include-metadata
        (is (contains? token-description# :last-update-time))
        (is (contains? token-description# :last-update-user))
        (is (contains? token-description# :owner))
        (is (contains? token-description# :root)))
      (is (= (cond-> {:health-check-url "/probe"
                      :name ~service-id-prefix
                      :owner (retrieve-username)}
               ~include-metadata (assoc :last-update-user (retrieve-username)
                                        :root ~token-root)
               (and ~deleted ~include-metadata) (assoc :deleted ~deleted))
             (dissoc token-description# :last-update-time :previous))))))

(deftest ^:parallel ^:integration-fast test-token-create-delete
  (testing-using-waiter-url
    (let [service-id-prefix (rand-name)
          token-prefix (create-token-name waiter-url service-id-prefix)
          {:keys [cookies]} (make-request waiter-url "/waiter-auth")
          num-tokens-to-create 10
          tokens-to-create (map #(str "token" %1 "." token-prefix) (range num-tokens-to-create))
          current-user (System/getProperty "user.name")
          token-root (retrieve-token-root waiter-url)]

      (log/info "creating the tokens")
      (doseq [token tokens-to-create]
        (let [response (post-token waiter-url {:health-check-url "/check"
                                               :name service-id-prefix
                                               :token token})]
          (assert-response-status response 200)
          (is (str/includes? (:body response) (str "Successfully created " token)))))

      (testing "update without etags"
        (doseq [token tokens-to-create]
          (let [response (post-token waiter-url {:health-check-url "/health"
                                                 :name service-id-prefix
                                                 :token token})]
            (assert-response-status response 200)
            (is (str/includes? (:body response) (str "Successfully updated " token))))))

      (testing "update with etags"
        (doseq [token tokens-to-create]
          (let [{:keys [body headers]} (get-token waiter-url token)
                token-description (-> body
                                      json/read-str
                                      walk/keywordize-keys
                                      (select-keys [:name])
                                      (assoc :health-check-url "/probe"
                                             :token token))
                response (post-token waiter-url token-description :headers {"if-match" (get headers "etag")})]
            (assert-response-status response 200)
            (is (str/includes? (:body response) (str "Successfully updated " token))))))

      (testing "update without changes"
        (doseq [token tokens-to-create]
          (let [{:keys [body headers]} (get-token waiter-url token)
                token-description (-> body
                                      json/read-str
                                      (select-keys sd/token-user-editable-keys)
                                      walk/keywordize-keys
                                      (assoc :token token))
                current-etag (get headers "etag")
                response (post-token waiter-url token-description :headers {"if-match" current-etag})]
            (assert-response-status response 200)
            (is (str/includes? (:body response) (str "No changes detected for " token)))
            (is (= current-etag (get-in response [:headers "etag"]))))))

      (testing "token retrieval - presence of etag header"
        (doseq [token tokens-to-create]
          (testing "via query parameters"
            (let [{:keys [headers] :as response} (get-token waiter-url token
                                                            :cookies cookies
                                                            :query-params {"include" "metadata"
                                                                           "token" token}
                                                            :request-headers {})
                  actual-etag (get headers "etag")]
              (assert-response-status response 200)
              (let [token-description (-> response :body json/read-str)]
                (is (= {"health-check-url" "/check", "name" service-id-prefix}
                       (-> token-description (get-in (repeat 2 "previous")) (select-keys sd/service-parameter-keys))))
                (is (= {"health-check-url" "/health", "name" service-id-prefix}
                       (-> token-description (get-in (repeat 1 "previous")) (select-keys sd/service-parameter-keys)))))
              (is actual-etag)))
          (testing "via x-waiter-token header"
            (let [{:keys [body headers] :as response} (get-token waiter-url token :cookies cookies)
                  actual-etag (get headers "etag")]
              (assert-response-status response 200)
              (is actual-etag)
              (when actual-etag
                (let [convert-last-update-time (fn [{:strs [last-update-time] :as service-description}]
                                                 (cond-> service-description
                                                   last-update-time
                                                   (assoc "last-update-time"
                                                          (-> last-update-time du/str-to-date .getMillis))))
                      expected-etag (str "E-"
                                         (-> body
                                             json/read-str
                                             convert-last-update-time
                                             (dissoc "previous")
                                             sd/parameters->id))]
                  (is (= expected-etag actual-etag))))))))

      (log/info "ensuring tokens can be retrieved and listed on each router")
      (doseq [token tokens-to-create]
        (doseq [[_ router-url] (routers waiter-url)]
          (-> (get-token router-url token :cookies cookies :query-params {"include" "none"})
              (assert-token-response token-root service-id-prefix false false))
          (-> (get-token router-url token :cookies cookies :query-params {"include" "metadata"})
              (assert-token-response token-root service-id-prefix false true))
          (let [{:keys [body] :as tokens-response}
                (list-tokens router-url current-user cookies {"include" ["deleted" "metadata"]})
                tokens (json/read-str body)]
            (assert-response-status tokens-response 200)
            (is (every? (fn [token-entry] (contains? token-entry "deleted")) tokens))
            (is (every? (fn [token-entry] (contains? token-entry "etag")) tokens))
            (is (some (fn [token-entry] (= token (get token-entry "token"))) tokens)))))

      (log/info "soft-deleting the tokens")
      (doseq [token tokens-to-create]
        (delete-token-and-assert waiter-url token :hard-delete false))

      (log/info "ensuring tokens can no longer be retrieved on each router without include=deleted parameter")
      (doseq [token tokens-to-create]
        (doseq [[router-id router-url] (routers waiter-url)]
          (let [router-state (kv-store-state router-url :cookies cookies)
                cache-data (get-in router-state ["state" "cache" "data"])
                token-cache-data (get cache-data (keyword token))]
            (is (nil? token-cache-data)
                (str token " data not nil (" token-cache-data ") on " router-id ", cache data =" cache-data)))
          (let [{:keys [body] :as response} (get-token router-url token :cookies cookies)]
            (assert-response-status response 404)
            (is (str/includes? (str body) "Couldn't find token") (str body)))
          (-> (get-token router-url token
                         :cookies cookies
                         :query-params {"include" ["deleted" "metadata"]})
              (assert-token-response token-root service-id-prefix true))
          (let [{:keys [body] :as tokens-response}
                (list-tokens router-url current-user cookies {"include" ["metadata"]})
                tokens (json/read-str body)]
            (assert-response-status tokens-response 200)
            (is (every? (fn [token-entry] (contains? token-entry "deleted")) tokens))
            (is (every? (fn [token-entry] (contains? token-entry "etag")) tokens))
            (is (not-any? (fn [token-entry] (= token (get token-entry "token"))) tokens)))
          (let [{:keys [body] :as tokens-response}
                (list-tokens router-url current-user cookies {"include" ["deleted" "metadata"]})
                tokens (json/read-str body)]
            (assert-response-status tokens-response 200)
            (is (every? (fn [token-entry] (contains? token-entry "deleted")) tokens))
            (is (every? (fn [token-entry] (contains? token-entry "etag")) tokens))
            (is (some (fn [token-entry] (= token (get token-entry "token"))) tokens)))))

      (log/info "hard-deleting the tokens")
      (doseq [token tokens-to-create]
        (delete-token-and-assert waiter-url token)
        (doseq [[_ router-url] (routers waiter-url)]
          (let [{:keys [body] :as response} (get-token router-url token :cookies cookies)]
            (assert-response-status response 404)
            (is (str/includes? (str body) "Couldn't find token") (str {:body body :token token})))
          (let [{:keys [body] :as tokens-response}
                (list-tokens router-url current-user cookies {"include" ["deleted" "metadata"]})
                token-entries (json/read-str body)]
            (assert-response-status tokens-response 200)
            (is (every? (fn [token-entry] (contains? token-entry "deleted")) token-entries))
            (is (every? (fn [token-entry] (contains? token-entry "etag")) token-entries))
            (comment
              ;; TODO Excluded as assertion is flaky
              (is (not-any? (fn [token-entry] (= token (get token-entry "token"))) token-entries)
                  (str token "entry found in list of deleted tokens!"
                       (->> token-entries (filter (fn [token-entry] (= token (get token-entry "token")))) vec)))))))

      (log/info "ensuring tokens can no longer be retrieved on each router with include=deleted parameter after hard-delete")
      (doseq [token tokens-to-create]
        (doseq [[_ router-url] (routers waiter-url)]
          (let [{:keys [body] :as response} (get-token router-url token
                                                       :cookies cookies
                                                       :query-params {"include" "deleted"})]
            (assert-response-status response 404)
            (is (str/includes? (str body) "Couldn't find token") (str body))))))))

(deftest ^:parallel ^:integration-fast ^:resource-heavy test-service-list-filtering
  (testing-using-waiter-url
    (let [service-name (rand-name)
          token-1 (create-token-name waiter-url (str "www." service-name ".t1"))
          token-2 (create-token-name waiter-url (str "www." service-name ".t2"))
          service-ids-atom (atom #{})
          token->version->etag-atom (atom {})
          all-tokens [token-1 token-2]
          all-version-suffixes ["v1" "v2" "v3"]
          router-id->router-url (routers waiter-url)
          {:keys [cookies]} (make-request waiter-url "/waiter-auth")]

      (doseq [version-suffix all-version-suffixes]
        (doseq [token all-tokens]
          (let [token-description (merge
                                    (kitchen-request-headers :prefix "")
                                    {:fallback-period-secs 0
                                     :name (str service-name "." (hash token))
                                     :token token
                                     :version (str service-name "." version-suffix)})
                {:keys [headers] :as response} (post-token waiter-url token-description)]
            (assert-response-status response 200)
            (let [token-etag (get headers "etag")]
              (log/info token "->" token-etag)
              (is (-> token-etag str/blank? not))
              (let [{:keys [service-id] :as response} (make-request-with-debug-info
                                                        {:x-waiter-token token}
                                                        #(make-request waiter-url "/environment" :headers %))]
                (assert-response-status response 200)
                (is (-> service-id str/blank? not))
                (swap! service-ids-atom conj service-id)
                ;; ensure all routers know about the service
                (doseq [[_ router-url] router-id->router-url]
                  (let [response (make-request-with-debug-info
                                   {:x-waiter-token token}
                                   #(make-request router-url "/environment" :headers % :cookies cookies))]
                    (assert-response-status response 200)
                    (is (= service-id (:service-id response))))))
              (swap! token->version->etag-atom assoc-in [token version-suffix] token-etag)))))
      (is (= (* (count all-tokens) (count all-version-suffixes)) (count @service-ids-atom))
          (str {:service-ids @service-ids-atom}))

      (testing "star in token filter"
        (doseq [[_ router-url] (routers waiter-url)]
          (let [query-params {"token" (str "www." service-name ".t*")}
                _ (log/info query-params)
                {:keys [body] :as response} (make-request router-url "/apps" :cookies cookies :query-params query-params)
                services (json/read-str body)
                service-tokens (mapcat (fn [entry]
                                         (some->> entry
                                                  walk/keywordize-keys
                                                  :source-tokens
                                                  flatten
                                                  (map :token)))
                                       services)]
            (assert-response-status response 200)
            (is (= (count @service-ids-atom) (count service-tokens))
                (str {:query-params query-params
                      :router-url router-url
                      :service-count (count services)
                      :service-tokens service-tokens})))))

      (doseq [loop-token all-tokens]
        (doseq [[_ router-url] (routers waiter-url)]
          (let [query-params {"token" loop-token}
                _ (log/info query-params)
                {:keys [body] :as response} (make-request router-url "/apps" :cookies cookies :query-params query-params)
                services (json/read-str body)
                service-tokens (mapcat (fn [entry]
                                         (some->> entry
                                                  walk/keywordize-keys
                                                  :source-tokens
                                                  flatten
                                                  (map :token)))
                                       services)]
            (assert-response-status response 200)
            (is (= 3 (count service-tokens))
                (str {:query-params query-params
                      :router-url router-url
                      :service-count (count services)
                      :service-tokens service-tokens}))))

        (doseq [version-suffix all-version-suffixes]
          (doseq [[_ router-url] (routers waiter-url)]
            (let [loop-etag (get-in @token->version->etag-atom [loop-token version-suffix])
                  query-params {"token-version" loop-etag}
                  _ (log/info query-params)
                  {:keys [body] :as response} (make-request router-url "/apps" :cookies cookies :query-params query-params)
                  services (json/read-str body)
                  service-token-versions (mapcat (fn [entry]
                                                   (some->> entry
                                                            walk/keywordize-keys
                                                            :source-tokens
                                                            flatten
                                                            (map :version)))
                                                 services)]
              (assert-response-status response 200)
              (is (= 1 (count service-token-versions))
                  (str {:query-params query-params
                        :router-url router-url
                        :service-count (count services)
                        :service-token-versions service-token-versions}))))))

      (doseq [service-id @service-ids-atom]
        (delete-service waiter-url service-id)))))

(deftest ^:parallel ^:integration-fast test-hostname-token
  (testing-using-waiter-url
    (let [service-id-prefix (rand-name)
          token-prefix (str "token-" (rand-int 3000000))
          token (create-token-name waiter-url token-prefix)
          token-root (retrieve-token-root waiter-url)]
      (testing "hostname-token-test"
        (try
          (log/info "basic hostname as token test")
          (let [current-user (retrieve-username)
                service-description (assoc (kitchen-request-headers)
                                           :x-waiter-metric-group token-prefix
                                           :x-waiter-permitted-user "*"
                                           :x-waiter-run-as-user current-user)]
            (testing "hostname-token-creation"
              (log/info "creating configuration using token" token)
              (let [{:keys [body status]}
                    (post-token waiter-url {:health-check-url "/probe"
                                            :metric-group token-prefix
                                            :name service-id-prefix
                                            :token token})]
                (when (not= 200 status)
                  (log/info "error:" body)
                  (is (not body))))
              (log/info "created configuration using token" token)
              (let [token-response (get-token waiter-url token)
                    response-body (json/read-str (:body token-response))]
                (is (contains? response-body "last-update-time"))
                (is (= {"health-check-url" "/probe"
                        "last-update-user" (retrieve-username)
                        "metric-group" token-prefix
                        "name" service-id-prefix
                        "owner" (retrieve-username)
                        "previous" {}
                        "root" token-root}
                       (dissoc response-body "last-update-time"))))
              (log/info "asserted retrieval of configuration for token" token))

            (testing "support-for-token-with-x-waiter-headers"
              (log/info "request with hostname token" token "along with x-waiter headers")
              (let [request-headers (merge service-description {"host" token})
                    path "/foo"
                    response (make-request waiter-url path :headers request-headers)
                    service-id (retrieve-service-id waiter-url (:request-headers response))]
                (assert-response-status response 200)
                (is (= (name-from-service-description waiter-url service-id) service-id-prefix))
                (is (= #{(make-source-tokens-entries waiter-url token)}
                       (service-id->source-tokens-entries waiter-url service-id)))
                (is (= token-prefix (service-id->metric-group waiter-url service-id))
                    (str {:service-id service-id :token token})))

              (log/info "request with hostname token" token "along with x-waiter headers except permitted-user")
              (let [request-headers (merge (dissoc service-description :x-waiter-permitted-user) {"host" token})
                    path "/foo"
                    response (make-request waiter-url path :headers request-headers)
                    service-id (retrieve-service-id waiter-url (:request-headers response))]
                (assert-response-status response 200)
                (is (= (name-from-service-description waiter-url service-id) service-id-prefix))
                (is (= #{(make-source-tokens-entries waiter-url token)}
                       (service-id->source-tokens-entries waiter-url service-id)))
                (is (= token-prefix (service-id->metric-group waiter-url service-id))
                    (str {:service-id service-id :token token}))
                ;; the above request hashes to a different service-id than the rest of the test, so we need to cleanup
                (delete-service waiter-url service-id))

              (log/info "request with hostname token" token "along with x-waiter headers except run-as-user")
              (let [request-headers (merge (dissoc service-description :x-waiter-run-as-user) {"host" token})
                    path "/foo"
                    response (make-request waiter-url path :headers request-headers)
                    service-id (retrieve-service-id waiter-url (:request-headers response))]
                (assert-response-status response 200)
                (is (= (name-from-service-description waiter-url service-id) service-id-prefix))
                (is (= #{(make-source-tokens-entries waiter-url token)}
                       (service-id->source-tokens-entries waiter-url service-id)))
                (is (= token-prefix (service-id->metric-group waiter-url service-id))
                    (str {:service-id service-id :token token}))

                (testing "backend request headers"
                  (let [{:keys [body] :as response} (make-request waiter-url "/request-info" :headers request-headers)
                        {:strs [headers]} (json/read-str (str body))]
                    (assert-response-status response 200)
                    (is (contains? headers "x-waiter-auth-principal"))
                    (is (contains? headers "x-waiter-authenticated-principal"))))

                ;; the above request hashes to a different service-id only when running as someone other than current-user
                ;; when the service-id is different, we need to cleanup
                (when (not= (System/getProperty "user.name") current-user)
                  (delete-service waiter-url service-id)))

              (log/info "request with hostname token" token "along with missing x-waiter headers except cmd")
              (let [request-headers (merge (dissoc service-description :x-waiter-cmd) {"host" token})
                    path "/foo"
                    {:keys [body] :as response} (make-request waiter-url path :headers request-headers)]
                (is (or (every? #(str/includes? body %)
                                ["Service description using waiter headers/token improperly configured",
                                 "{\"cmd\" missing-required-key}"])
                        (str/includes? body "Invalid command or version"))
                    (str "response body was: " response))
                (assert-response-status response 400))

              (log/info "request with hostname token and x-waiter-debug token" token "along with x-waiter headers")
              (let [request-headers (merge service-description {:x-waiter-debug "true", "host" token})
                    path "/foo"
                    {:keys [headers request-headers] :as response} (make-request waiter-url path :headers request-headers)
                    service-id (retrieve-service-id waiter-url request-headers)]
                (assert-response-status response 200)
                (is (= (name-from-service-description waiter-url service-id) service-id-prefix))
                (is (= #{(make-source-tokens-entries waiter-url token)}
                       (service-id->source-tokens-entries waiter-url service-id)))
                (is (every? #(not (str/blank? (get headers %)))
                            (concat required-response-headers (retrieve-debug-response-headers waiter-url)))
                    (str headers))
                (is (= token-prefix (service-id->metric-group waiter-url service-id))
                    (str {:service-id service-id :token token}))
                (delete-service waiter-url service-id))))
          (finally
            (delete-token-and-assert waiter-url token)))))))

(deftest ^:parallel ^:integration-fast test-token-administer-unaffected-by-run-as-user-permissions
  (testing-using-waiter-url
    (let [service-id-prefix (rand-name)
          token-root (retrieve-token-root waiter-url)]
      (testing "token-administering"
        (testing "active-token"
          (let [last-update-time (System/currentTimeMillis)
                token (create-token-name waiter-url service-id-prefix)]

            (let [token-description {:health-check-url "/probe"
                                     :name service-id-prefix
                                     :owner (retrieve-username)
                                     :run-as-user (retrieve-username)
                                     :token token}
                  response (post-token waiter-url token-description)]
              (assert-response-status response 200))

            (testing "if-match-required during update"
              (let [token-description {:health-check-url "/probe-1"
                                       :last-update-time last-update-time
                                       :name service-id-prefix
                                       :owner (retrieve-username)
                                       :token token}
                    {:keys [body] :as response} (post-token waiter-url token-description
                                                            :headers {}
                                                            :query-params {"update-mode" "admin"})]
                (assert-response-status response 400)
                (is (str/includes? body "Must specify if-match header for admin mode token updates"))))

            (try
              (testing "successful admin mode update"
                (log/info "creating configuration using token" token)
                (let [{:keys [body headers]} (get-token waiter-url token)
                      token-description (-> body
                                            json/read-str
                                            walk/keywordize-keys
                                            (assoc :health-check-url "/probe-2"
                                                   :last-update-time last-update-time
                                                   :owner (retrieve-username)
                                                   :run-as-user "i-do-not-exist-but-will-not-be-checked"
                                                   :token token))
                      response (post-token waiter-url token-description
                                           :headers {"if-match" (get headers "etag")}
                                           :query-params {"update-mode" "admin"})]
                  (assert-response-status response 200)
                  (log/info "created configuration using token" token))

                (let [token-response (get-token waiter-url token)
                      response-body (json/read-str (:body token-response))]
                  (is (= {"health-check-url" "/probe-2"
                          "last-update-time" (-> last-update-time DateTime. du/date-to-str)
                          "last-update-user" (retrieve-username)
                          "name" service-id-prefix,
                          "owner" (retrieve-username)
                          "root" token-root
                          "run-as-user" "i-do-not-exist-but-will-not-be-checked"}
                         (dissoc response-body "previous")))
                  (log/info "asserted retrieval of configuration for token" token)))

              (testing "update with invalid etag"
                (let [token-description {:health-check-url "/probe-3"
                                         :last-update-time last-update-time
                                         :name service-id-prefix
                                         :owner (retrieve-username)
                                         :run-as-user "foo-bar"
                                         :token token}
                      {:keys [body] :as response} (post-token waiter-url token-description
                                                              :headers {"host" token
                                                                        "if-match" "1010"}
                                                              :query-params {"update-mode" "admin"})]
                  (assert-response-status response 412)
                  (is (str/includes? (str body) "Cannot modify stale token"))))

              (testing "hard-delete without etag"
                (let [{:keys [body] :as response} (make-request waiter-url "/token"
                                                                :headers {"host" token}
                                                                :method :delete
                                                                :query-params {"hard-delete" true})]
                  (assert-response-status response 400)
                  (is (str/includes? (str body) "Must specify if-match header for token hard deletes"))))

              (testing "hard-delete with invalid etag"
                (let [{:keys [body] :as response} (make-request waiter-url "/token"
                                                                :headers {"host" token
                                                                          "if-match" "1010"}
                                                                :method :delete
                                                                :query-params {"hard-delete" true})]
                  (assert-response-status response 412)
                  (is (str/includes? (str body) "Cannot modify stale token"))))

              (finally
                (delete-token-and-assert waiter-url token)))))

        (testing "deleted-token"
          (let [last-update-time (System/currentTimeMillis)
                token (create-token-name waiter-url service-id-prefix)]
            (try
              (log/info "creating configuration using token" token)
              (let [token-description {:health-check-url "/probe-3"
                                       :name service-id-prefix
                                       :owner (retrieve-username)
                                       :token token}
                    response (post-token waiter-url token-description :headers {"host" token})]
                (assert-response-status response 200))

              (let [{:keys [body headers]} (get-token waiter-url token)
                    token-description (-> body
                                          json/read-str
                                          walk/keywordize-keys
                                          (assoc :deleted true
                                                 :health-check-url "/probe"
                                                 :last-update-time last-update-time
                                                 :run-as-user "foo-bar"
                                                 :token token))
                    response (post-token waiter-url token-description
                                         :headers {"if-match" (get headers "etag")}
                                         :query-params {"update-mode" "admin"})]
                (assert-response-status response 200))
              (log/info "created configuration using token" token)
              (let [{:keys [body] :as response} (get-token waiter-url token)]
                (assert-response-status response 404)
                (is (str/includes? (str body) "Couldn't find token") (str body)))
              (let [token-response (get-token waiter-url token :query-params {"include" ["deleted" "metadata"]})
                    response-body (json/read-str (:body token-response))]
                (is (= {"deleted" true
                        "health-check-url" "/probe"
                        "last-update-time" (-> last-update-time DateTime. du/date-to-str)
                        "last-update-user" (retrieve-username)
                        "name" service-id-prefix
                        "owner" (retrieve-username)
                        "root" token-root
                        "run-as-user" "foo-bar"}
                       (dissoc response-body "previous"))))
              (log/info "asserted retrieval of configuration for token" token)
              (finally
                (delete-token-and-assert waiter-url token)))))))))

(deftest ^:parallel ^:integration-fast test-named-token
  (testing-using-waiter-url
    (log/info "basic named token test")
    (let [service-id-prefix (rand-name)
          token (create-token-name waiter-url service-id-prefix)]
      (try
        (log/info "creating configuration using token" token)
        (let [token-definition (assoc
                                 (kitchen-request-headers :prefix "")
                                 :name service-id-prefix
                                 :token token)
              {:keys [body status]} (post-token waiter-url token-definition)]
          (when (not= 200 status)
            (log/info "error:" body)
            (is (not body))))
        (log/info "created configuration using token" token)
        (log/info "retrieving configuration for token" token)
        (let [token-response (get-token waiter-url token)
              response-body (str (:body token-response))]
          (when (not (str/includes? response-body service-id-prefix))
            (log/info response-body))
          (assert-response-status token-response 200)
          (is (str/includes? response-body service-id-prefix)))
        (log/info "asserted retrieval of configuration for token" token)

        (log/info "making Waiter request with token" token "in header")
        (let [request-headers {:x-waiter-token token}
              path "/foo"
              response (make-request waiter-url path :headers request-headers)
              service-id (retrieve-service-id waiter-url (:request-headers response))]
          (assert-response-status response 200)
          (is (= (name-from-service-description waiter-url service-id) service-id-prefix))
          (is (= #{(make-source-tokens-entries waiter-url token)}
                 (service-id->source-tokens-entries waiter-url service-id))))

        (log/info "making Waiter request with token and x-waiter-debug token" token "in header")
        (let [request-headers {:x-waiter-debug "true", :x-waiter-token token}
              path "/foo"
              {:keys [headers request-headers] :as response} (make-request waiter-url path :headers request-headers)
              service-id (retrieve-service-id waiter-url request-headers)]
          (assert-response-status response 200)
          (is (= (name-from-service-description waiter-url service-id) service-id-prefix))
          (is (= #{(make-source-tokens-entries waiter-url token)}
                 (service-id->source-tokens-entries waiter-url service-id)))
          (is (every? #(not (str/blank? (get headers %)))
                      (concat required-response-headers (retrieve-debug-response-headers waiter-url)))
              (str headers))
          (delete-service waiter-url service-id))
        (finally
          (delete-token-and-assert waiter-url token))))))

(deftest ^:parallel ^:integration-fast test-star-run-as-user-token
  (testing-using-waiter-url
    (let [service-id-prefix (rand-name)
          token (create-token-name waiter-url service-id-prefix)]
      (try
        (log/info "creating configuration using token" token)
        (let [token-definition (assoc
                                 (kitchen-request-headers :prefix "")
                                 :name service-id-prefix
                                 :run-as-user "*"
                                 :token token)
              {:keys [body status]} (post-token waiter-url token-definition)]
          (when (not= 200 status)
            (log/info "error:" body)
            (is (not body))))
        (log/info "created configuration using token" token)
        (log/info "retrieving configuration for token" token)
        (let [token-response (get-token waiter-url token)
              response-body (str (:body token-response))]
          (when (not (str/includes? response-body service-id-prefix))
            (log/info response-body))
          (assert-response-status token-response 200)
          (is (str/includes? response-body service-id-prefix)))
        (log/info "asserted retrieval of configuration for token" token)

        (let [{:keys [body] :as token-response} (get-token waiter-url token)
              token-description (try (json/read-str (str body))
                                     (catch Exception _
                                       (is false (str "Failed to parse token" body))))]
          (assert-response-status token-response 200)
          (is (= "/status" (token-description "health-check-url")))
          (is (= service-id-prefix (token-description "name")))
          (is (= "*" (token-description "run-as-user")))
          (is (System/getProperty "user.name") (token-description "owner")))

        (log/info "making Waiter request with token" token "in header")
        (let [request-headers {:x-waiter-token token}
              path "/foo"
              response (make-request waiter-url path :headers request-headers)
              service-id (retrieve-service-id waiter-url (:request-headers response))]
          (assert-response-status response 200)
          (is (= (name-from-service-description waiter-url service-id) service-id-prefix))
          (is (= #{(make-source-tokens-entries waiter-url token)}
                 (service-id->source-tokens-entries waiter-url service-id)))
          (is (= (retrieve-username) (:run-as-user (service-id->service-description waiter-url service-id)))))

        (log/info "making Waiter request with token and x-waiter-debug token" token "in header")
        (let [request-headers {:x-waiter-debug "true", :x-waiter-token token}
              path "/foo"
              {:keys [headers request-headers] :as response} (make-request waiter-url path :headers request-headers)
              service-id (retrieve-service-id waiter-url request-headers)]
          (assert-response-status response 200)
          (is (= (name-from-service-description waiter-url service-id) service-id-prefix))
          (is (= #{(make-source-tokens-entries waiter-url token)}
                 (service-id->source-tokens-entries waiter-url service-id)))
          (is (every? #(not (str/blank? (get headers %)))
                      (concat required-response-headers (retrieve-debug-response-headers waiter-url)))
              (str headers))
          (delete-service waiter-url service-id))
        (finally
          (delete-token-and-assert waiter-url token))))))

(deftest ^:parallel ^:integration-fast ^:explicit test-on-the-fly-to-token
  (testing-using-waiter-url
    (let [name-string (rand-name)
          canary-response (make-kitchen-request waiter-url {:x-waiter-name name-string})
          service-id-1 (retrieve-service-id waiter-url (:request-headers canary-response))]
      (with-service-cleanup
        service-id-1
        (is (str/includes? service-id-1 name-string) (str "ERROR: App-name is missing " name-string))
        (is (nil? (service-id->source-tokens-entries waiter-url service-id-1)))
        (let [token (str "^SERVICE-ID#" service-id-1)
              response (make-request-with-debug-info {:x-waiter-token token} #(make-request waiter-url "" :headers %))
              service-id-2 (:service-id response)]
          (with-service-cleanup
            service-id-2
            (assert-response-status response 200)
            (is service-id-2)
            (is (= service-id-1 service-id-2) "The on-the-fly and token-based service ids do not match")
            (is (= #{(make-source-tokens-entries waiter-url token)}
                   (service-id->source-tokens-entries waiter-url service-id-2)))))))))

(deftest ^:parallel ^:integration-fast test-bad-token
  (testing-using-waiter-url

    (testing "ignore missing token when have service description"
      (let [response (make-request waiter-url "/pathabc" :headers {"host" "missing_token" "X-Waiter-CPUS" "1"})]
        (is (str/includes? (:body response) "Service description using waiter headers/token improperly configured"))
        (assert-response-status response 400)))

    (testing "ignore invalid token when have service description"
      (let [response (make-request waiter-url "/pathabc" :headers {"host" "bad/token" "X-Waiter-CPUS" "1"})]
        (is (str/includes? (:body response) "Service description using waiter headers/token improperly configured"))
        (assert-response-status response 400)))

    (testing "ignore invalid token when have invalid service description"
      (let [response (make-request waiter-url "/pathabc" :headers {"host" "bad/token" "X-Waiter-CPUS" "one"})]
        (is (str/includes? (:body response) "Service description using waiter headers/token improperly configured"))
        (assert-response-status response 400)))<commit_msg>uses commands in incomplete service description token tests (#493)<commit_after>;;
;; Copyright (c) Two Sigma Open Source, LLC
;;
;; Licensed under the Apache License, Version 2.0 (the "License");
;; you may not use this file except in compliance with the License.
;; You may obtain a copy of the License at
;;
;;  http://www.apache.org/licenses/LICENSE-2.0
;;
;; Unless required by applicable law or agreed to in writing, software
;; distributed under the License is distributed on an "AS IS" BASIS,
;; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
;; See the License for the specific language governing permissions and
;; limitations under the License.
;;
(ns waiter.token-request-test
  (:require [clj-time.core :as t]
            [clojure.core.async :as async]
            [clojure.data.json :as json]
            [clojure.string :as str]
            [clojure.test :refer :all]
            [clojure.tools.logging :as log]
            [clojure.walk :as walk]
            [plumbing.core :as pc]
            [waiter.service-description :as sd]
            [waiter.util.client-tools :refer :all]
            [waiter.util.date-utils :as du])
  (:import (java.net URL)
           (org.joda.time DateTime)))

(defn- waiter-url->token-url
  [waiter-url]
  (str HTTP-SCHEME waiter-url "/token"))

(defn- retrieve-token-root
  [waiter-url]
  (-> (waiter-settings waiter-url)
      (get-in [:cluster-config :name])))

(deftest ^:parallel ^:integration-fast test-update-token-cache-consistency
  (testing-using-waiter-url
    (let [service-id-prefix (rand-name "testhostname")
          token (rand-name)
          update-token-fn (fn [version]
                            (let [{:keys [status]} (post-token waiter-url {:name service-id-prefix
                                                                           :cpus 1
                                                                           :mem 1250
                                                                           :version version
                                                                           :cmd "test command"
                                                                           :health-check-url "/ping"
                                                                           :permitted-user "*"
                                                                           :run-as-user (retrieve-username)
                                                                           :token token})]
                              (is (= 200 status))))
          validate-token-fn (fn [version num-threads num-iter]
                              (parallelize-requests
                                num-threads
                                num-iter
                                (fn []
                                  (let [{:keys [body status]} (get-token waiter-url token)]
                                    (is (= 200 status))
                                    (is (every? #(str/includes? (str body) (str %)) [service-id-prefix "1250" version "test command"]))
                                    (is (not-any? #(str/includes? (str body) (str %)) ["invalid"]))))))
          update-and-validate-token-fn (fn [version]
                                         (log/info "creating configuration using token" token "with version" version)
                                         (update-token-fn version)
                                         (log/info "asserting configuration for token" token "from routers (best-effort)")
                                         (validate-token-fn version 10 10))
          version1 "123987132937213712"
          version2 "656760465406467480"
          version3 "678219671796032121"]
      (try
        (update-and-validate-token-fn version1)
        (update-and-validate-token-fn version2)
        (update-and-validate-token-fn version3)
        (finally
          (delete-token-and-assert waiter-url token))))))

(defn- name-from-service-description [waiter-url service-id]
  (get-in (service-settings waiter-url service-id) [:service-description :name]))

(defn- service-id->source-tokens-entries [waiter-url service-id]
  (some-> (service-settings waiter-url service-id)
          :source-tokens
          walk/stringify-keys
          set))

(defn- token->etag [waiter-url token]
  (-> (get-token waiter-url token :query-params {"token" token})
      :headers
      (get "etag")))

(defn- make-source-tokens-entries [waiter-url & tokens]
  (mapv (fn [token] {"token" token "version" (token->etag waiter-url token)}) tokens))

(defn- create-token-name
  [waiter-url service-id-prefix]
  (str service-id-prefix "." (subs waiter-url 0 (str/index-of waiter-url ":"))))

(defn- list-tokens
  [waiter-url owner cookies query-params]
  (let [tokens-response (make-request waiter-url "/tokens" :cookies cookies :query-params query-params)]
    (log/debug "retrieved tokens for owner " owner ":" (:body tokens-response))
    tokens-response))

(defn- parse-token-description
  "Parses a response as json and keywordizes the map."
  [response-body]
  (try
    (-> response-body str json/read-str pc/keywordize-map)
    (catch Exception _
      (is false (str "Failed to parse token " response-body)))))

(defmacro assert-token-response
  "Asserts the token data in the response"
  ([response token-root service-id-prefix deleted]
   `(assert-token-response ~response ~token-root ~service-id-prefix ~deleted true))
  ([response token-root service-id-prefix deleted include-metadata]
   `(let [body# (:body ~response)
          token-description# (parse-token-description body#)]
      (assert-response-status ~response 200)
      (when ~include-metadata
        (is (contains? token-description# :last-update-time))
        (is (contains? token-description# :last-update-user))
        (is (contains? token-description# :owner))
        (is (contains? token-description# :root)))
      (is (= (cond-> {:health-check-url "/probe"
                      :name ~service-id-prefix
                      :owner (retrieve-username)}
               ~include-metadata (assoc :last-update-user (retrieve-username)
                                        :root ~token-root)
               (and ~deleted ~include-metadata) (assoc :deleted ~deleted))
             (dissoc token-description# :last-update-time :previous))))))

(deftest ^:parallel ^:integration-fast test-token-create-delete
  (testing-using-waiter-url
    (let [service-id-prefix (rand-name)
          token-prefix (create-token-name waiter-url service-id-prefix)
          {:keys [cookies]} (make-request waiter-url "/waiter-auth")
          num-tokens-to-create 10
          tokens-to-create (map #(str "token" %1 "." token-prefix) (range num-tokens-to-create))
          current-user (System/getProperty "user.name")
          token-root (retrieve-token-root waiter-url)]

      (log/info "creating the tokens")
      (doseq [token tokens-to-create]
        (let [response (post-token waiter-url {:health-check-url "/check"
                                               :name service-id-prefix
                                               :token token})]
          (assert-response-status response 200)
          (is (str/includes? (:body response) (str "Successfully created " token)))))

      (testing "update without etags"
        (doseq [token tokens-to-create]
          (let [response (post-token waiter-url {:health-check-url "/health"
                                                 :name service-id-prefix
                                                 :token token})]
            (assert-response-status response 200)
            (is (str/includes? (:body response) (str "Successfully updated " token))))))

      (testing "update with etags"
        (doseq [token tokens-to-create]
          (let [{:keys [body headers]} (get-token waiter-url token)
                token-description (-> body
                                      json/read-str
                                      walk/keywordize-keys
                                      (select-keys [:name])
                                      (assoc :health-check-url "/probe"
                                             :token token))
                response (post-token waiter-url token-description :headers {"if-match" (get headers "etag")})]
            (assert-response-status response 200)
            (is (str/includes? (:body response) (str "Successfully updated " token))))))

      (testing "update without changes"
        (doseq [token tokens-to-create]
          (let [{:keys [body headers]} (get-token waiter-url token)
                token-description (-> body
                                      json/read-str
                                      (select-keys sd/token-user-editable-keys)
                                      walk/keywordize-keys
                                      (assoc :token token))
                current-etag (get headers "etag")
                response (post-token waiter-url token-description :headers {"if-match" current-etag})]
            (assert-response-status response 200)
            (is (str/includes? (:body response) (str "No changes detected for " token)))
            (is (= current-etag (get-in response [:headers "etag"]))))))

      (testing "token retrieval - presence of etag header"
        (doseq [token tokens-to-create]
          (testing "via query parameters"
            (let [{:keys [headers] :as response} (get-token waiter-url token
                                                            :cookies cookies
                                                            :query-params {"include" "metadata"
                                                                           "token" token}
                                                            :request-headers {})
                  actual-etag (get headers "etag")]
              (assert-response-status response 200)
              (let [token-description (-> response :body json/read-str)]
                (is (= {"health-check-url" "/check", "name" service-id-prefix}
                       (-> token-description (get-in (repeat 2 "previous")) (select-keys sd/service-parameter-keys))))
                (is (= {"health-check-url" "/health", "name" service-id-prefix}
                       (-> token-description (get-in (repeat 1 "previous")) (select-keys sd/service-parameter-keys)))))
              (is actual-etag)))
          (testing "via x-waiter-token header"
            (let [{:keys [body headers] :as response} (get-token waiter-url token :cookies cookies)
                  actual-etag (get headers "etag")]
              (assert-response-status response 200)
              (is actual-etag)
              (when actual-etag
                (let [convert-last-update-time (fn [{:strs [last-update-time] :as service-description}]
                                                 (cond-> service-description
                                                   last-update-time
                                                   (assoc "last-update-time"
                                                          (-> last-update-time du/str-to-date .getMillis))))
                      expected-etag (str "E-"
                                         (-> body
                                             json/read-str
                                             convert-last-update-time
                                             (dissoc "previous")
                                             sd/parameters->id))]
                  (is (= expected-etag actual-etag))))))))

      (log/info "ensuring tokens can be retrieved and listed on each router")
      (doseq [token tokens-to-create]
        (doseq [[_ router-url] (routers waiter-url)]
          (-> (get-token router-url token :cookies cookies :query-params {"include" "none"})
              (assert-token-response token-root service-id-prefix false false))
          (-> (get-token router-url token :cookies cookies :query-params {"include" "metadata"})
              (assert-token-response token-root service-id-prefix false true))
          (let [{:keys [body] :as tokens-response}
                (list-tokens router-url current-user cookies {"include" ["deleted" "metadata"]})
                tokens (json/read-str body)]
            (assert-response-status tokens-response 200)
            (is (every? (fn [token-entry] (contains? token-entry "deleted")) tokens))
            (is (every? (fn [token-entry] (contains? token-entry "etag")) tokens))
            (is (some (fn [token-entry] (= token (get token-entry "token"))) tokens)))))

      (log/info "soft-deleting the tokens")
      (doseq [token tokens-to-create]
        (delete-token-and-assert waiter-url token :hard-delete false))

      (log/info "ensuring tokens can no longer be retrieved on each router without include=deleted parameter")
      (doseq [token tokens-to-create]
        (doseq [[router-id router-url] (routers waiter-url)]
          (let [router-state (kv-store-state router-url :cookies cookies)
                cache-data (get-in router-state ["state" "cache" "data"])
                token-cache-data (get cache-data (keyword token))]
            (is (nil? token-cache-data)
                (str token " data not nil (" token-cache-data ") on " router-id ", cache data =" cache-data)))
          (let [{:keys [body] :as response} (get-token router-url token :cookies cookies)]
            (assert-response-status response 404)
            (is (str/includes? (str body) "Couldn't find token") (str body)))
          (-> (get-token router-url token
                         :cookies cookies
                         :query-params {"include" ["deleted" "metadata"]})
              (assert-token-response token-root service-id-prefix true))
          (let [{:keys [body] :as tokens-response}
                (list-tokens router-url current-user cookies {"include" ["metadata"]})
                tokens (json/read-str body)]
            (assert-response-status tokens-response 200)
            (is (every? (fn [token-entry] (contains? token-entry "deleted")) tokens))
            (is (every? (fn [token-entry] (contains? token-entry "etag")) tokens))
            (is (not-any? (fn [token-entry] (= token (get token-entry "token"))) tokens)))
          (let [{:keys [body] :as tokens-response}
                (list-tokens router-url current-user cookies {"include" ["deleted" "metadata"]})
                tokens (json/read-str body)]
            (assert-response-status tokens-response 200)
            (is (every? (fn [token-entry] (contains? token-entry "deleted")) tokens))
            (is (every? (fn [token-entry] (contains? token-entry "etag")) tokens))
            (is (some (fn [token-entry] (= token (get token-entry "token"))) tokens)))))

      (log/info "hard-deleting the tokens")
      (doseq [token tokens-to-create]
        (delete-token-and-assert waiter-url token)
        (doseq [[_ router-url] (routers waiter-url)]
          (let [{:keys [body] :as response} (get-token router-url token :cookies cookies)]
            (assert-response-status response 404)
            (is (str/includes? (str body) "Couldn't find token") (str {:body body :token token})))
          (let [{:keys [body] :as tokens-response}
                (list-tokens router-url current-user cookies {"include" ["deleted" "metadata"]})
                token-entries (json/read-str body)]
            (assert-response-status tokens-response 200)
            (is (every? (fn [token-entry] (contains? token-entry "deleted")) token-entries))
            (is (every? (fn [token-entry] (contains? token-entry "etag")) token-entries))
            (comment
              ;; TODO Excluded as assertion is flaky
              (is (not-any? (fn [token-entry] (= token (get token-entry "token"))) token-entries)
                  (str token "entry found in list of deleted tokens!"
                       (->> token-entries (filter (fn [token-entry] (= token (get token-entry "token")))) vec)))))))

      (log/info "ensuring tokens can no longer be retrieved on each router with include=deleted parameter after hard-delete")
      (doseq [token tokens-to-create]
        (doseq [[_ router-url] (routers waiter-url)]
          (let [{:keys [body] :as response} (get-token router-url token
                                                       :cookies cookies
                                                       :query-params {"include" "deleted"})]
            (assert-response-status response 404)
            (is (str/includes? (str body) "Couldn't find token") (str body))))))))

(deftest ^:parallel ^:integration-fast ^:resource-heavy test-service-list-filtering
  (testing-using-waiter-url
    (let [service-name (rand-name)
          token-1 (create-token-name waiter-url (str "www." service-name ".t1"))
          token-2 (create-token-name waiter-url (str "www." service-name ".t2"))
          service-ids-atom (atom #{})
          token->version->etag-atom (atom {})
          all-tokens [token-1 token-2]
          all-version-suffixes ["v1" "v2" "v3"]
          router-id->router-url (routers waiter-url)
          {:keys [cookies]} (make-request waiter-url "/waiter-auth")]

      (doseq [version-suffix all-version-suffixes]
        (doseq [token all-tokens]
          (let [token-description (merge
                                    (kitchen-request-headers :prefix "")
                                    {:fallback-period-secs 0
                                     :name (str service-name "." (hash token))
                                     :token token
                                     :version (str service-name "." version-suffix)})
                {:keys [headers] :as response} (post-token waiter-url token-description)]
            (assert-response-status response 200)
            (let [token-etag (get headers "etag")]
              (log/info token "->" token-etag)
              (is (-> token-etag str/blank? not))
              (let [{:keys [service-id] :as response} (make-request-with-debug-info
                                                        {:x-waiter-token token}
                                                        #(make-request waiter-url "/environment" :headers %))]
                (assert-response-status response 200)
                (is (-> service-id str/blank? not))
                (swap! service-ids-atom conj service-id)
                ;; ensure all routers know about the service
                (doseq [[_ router-url] router-id->router-url]
                  (let [response (make-request-with-debug-info
                                   {:x-waiter-token token}
                                   #(make-request router-url "/environment" :headers % :cookies cookies))]
                    (assert-response-status response 200)
                    (is (= service-id (:service-id response))))))
              (swap! token->version->etag-atom assoc-in [token version-suffix] token-etag)))))
      (is (= (* (count all-tokens) (count all-version-suffixes)) (count @service-ids-atom))
          (str {:service-ids @service-ids-atom}))

      (testing "star in token filter"
        (doseq [[_ router-url] (routers waiter-url)]
          (let [query-params {"token" (str "www." service-name ".t*")}
                _ (log/info query-params)
                {:keys [body] :as response} (make-request router-url "/apps" :cookies cookies :query-params query-params)
                services (json/read-str body)
                service-tokens (mapcat (fn [entry]
                                         (some->> entry
                                                  walk/keywordize-keys
                                                  :source-tokens
                                                  flatten
                                                  (map :token)))
                                       services)]
            (assert-response-status response 200)
            (is (= (count @service-ids-atom) (count service-tokens))
                (str {:query-params query-params
                      :router-url router-url
                      :service-count (count services)
                      :service-tokens service-tokens})))))

      (doseq [loop-token all-tokens]
        (doseq [[_ router-url] (routers waiter-url)]
          (let [query-params {"token" loop-token}
                _ (log/info query-params)
                {:keys [body] :as response} (make-request router-url "/apps" :cookies cookies :query-params query-params)
                services (json/read-str body)
                service-tokens (mapcat (fn [entry]
                                         (some->> entry
                                                  walk/keywordize-keys
                                                  :source-tokens
                                                  flatten
                                                  (map :token)))
                                       services)]
            (assert-response-status response 200)
            (is (= 3 (count service-tokens))
                (str {:query-params query-params
                      :router-url router-url
                      :service-count (count services)
                      :service-tokens service-tokens}))))

        (doseq [version-suffix all-version-suffixes]
          (doseq [[_ router-url] (routers waiter-url)]
            (let [loop-etag (get-in @token->version->etag-atom [loop-token version-suffix])
                  query-params {"token-version" loop-etag}
                  _ (log/info query-params)
                  {:keys [body] :as response} (make-request router-url "/apps" :cookies cookies :query-params query-params)
                  services (json/read-str body)
                  service-token-versions (mapcat (fn [entry]
                                                   (some->> entry
                                                            walk/keywordize-keys
                                                            :source-tokens
                                                            flatten
                                                            (map :version)))
                                                 services)]
              (assert-response-status response 200)
              (is (= 1 (count service-token-versions))
                  (str {:query-params query-params
                        :router-url router-url
                        :service-count (count services)
                        :service-token-versions service-token-versions}))))))

      (doseq [service-id @service-ids-atom]
        (delete-service waiter-url service-id)))))

(deftest ^:parallel ^:integration-fast test-hostname-token
  (testing-using-waiter-url
    (let [service-id-prefix (rand-name)
          token-prefix (str "token-" (rand-int 3000000))
          token (create-token-name waiter-url token-prefix)
          token-root (retrieve-token-root waiter-url)]
      (testing "hostname-token-test"
        (try
          (log/info "basic hostname as token test")
          (let [current-user (retrieve-username)
                service-description (assoc (kitchen-request-headers)
                                           :x-waiter-metric-group token-prefix
                                           :x-waiter-permitted-user "*"
                                           :x-waiter-run-as-user current-user)]
            (testing "hostname-token-creation"
              (log/info "creating configuration using token" token)
              (let [{:keys [body status]}
                    (post-token waiter-url {:health-check-url "/probe"
                                            :metric-group token-prefix
                                            :name service-id-prefix
                                            :token token})]
                (when (not= 200 status)
                  (log/info "error:" body)
                  (is (not body))))
              (log/info "created configuration using token" token)
              (let [token-response (get-token waiter-url token)
                    response-body (json/read-str (:body token-response))]
                (is (contains? response-body "last-update-time"))
                (is (= {"health-check-url" "/probe"
                        "last-update-user" (retrieve-username)
                        "metric-group" token-prefix
                        "name" service-id-prefix
                        "owner" (retrieve-username)
                        "previous" {}
                        "root" token-root}
                       (dissoc response-body "last-update-time"))))
              (log/info "asserted retrieval of configuration for token" token))

            (testing "support-for-token-with-x-waiter-headers"
              (log/info "request with hostname token" token "along with x-waiter headers")
              (let [request-headers (merge service-description {"host" token})
                    path "/foo"
                    response (make-request waiter-url path :headers request-headers)
                    service-id (retrieve-service-id waiter-url (:request-headers response))]
                (assert-response-status response 200)
                (is (= (name-from-service-description waiter-url service-id) service-id-prefix))
                (is (= #{(make-source-tokens-entries waiter-url token)}
                       (service-id->source-tokens-entries waiter-url service-id)))
                (is (= token-prefix (service-id->metric-group waiter-url service-id))
                    (str {:service-id service-id :token token})))

              (log/info "request with hostname token" token "along with x-waiter headers except permitted-user")
              (let [request-headers (merge (dissoc service-description :x-waiter-permitted-user) {"host" token})
                    path "/foo"
                    response (make-request waiter-url path :headers request-headers)
                    service-id (retrieve-service-id waiter-url (:request-headers response))]
                (assert-response-status response 200)
                (is (= (name-from-service-description waiter-url service-id) service-id-prefix))
                (is (= #{(make-source-tokens-entries waiter-url token)}
                       (service-id->source-tokens-entries waiter-url service-id)))
                (is (= token-prefix (service-id->metric-group waiter-url service-id))
                    (str {:service-id service-id :token token}))
                ;; the above request hashes to a different service-id than the rest of the test, so we need to cleanup
                (delete-service waiter-url service-id))

              (log/info "request with hostname token" token "along with x-waiter headers except run-as-user")
              (let [request-headers (merge (dissoc service-description :x-waiter-run-as-user) {"host" token})
                    path "/foo"
                    response (make-request waiter-url path :headers request-headers)
                    service-id (retrieve-service-id waiter-url (:request-headers response))]
                (assert-response-status response 200)
                (is (= (name-from-service-description waiter-url service-id) service-id-prefix))
                (is (= #{(make-source-tokens-entries waiter-url token)}
                       (service-id->source-tokens-entries waiter-url service-id)))
                (is (= token-prefix (service-id->metric-group waiter-url service-id))
                    (str {:service-id service-id :token token}))

                (testing "backend request headers"
                  (let [{:keys [body] :as response} (make-request waiter-url "/request-info" :headers request-headers)
                        {:strs [headers]} (json/read-str (str body))]
                    (assert-response-status response 200)
                    (is (contains? headers "x-waiter-auth-principal"))
                    (is (contains? headers "x-waiter-authenticated-principal"))))

                ;; the above request hashes to a different service-id only when running as someone other than current-user
                ;; when the service-id is different, we need to cleanup
                (when (not= (System/getProperty "user.name") current-user)
                  (delete-service waiter-url service-id)))

              (log/info "request with hostname token" token "along with missing x-waiter headers except cmd")
              (let [request-headers (merge (dissoc service-description :x-waiter-cmd) {"host" token})
                    path "/foo"
                    {:keys [body] :as response} (make-request waiter-url path :headers request-headers)]
                (is (or (every? #(str/includes? body %)
                                ["Service description using waiter headers/token improperly configured",
                                 "{\"cmd\" missing-required-key}"])
                        (str/includes? body "Invalid command or version"))
                    (str "response body was: " response))
                (assert-response-status response 400))

              (log/info "request with hostname token and x-waiter-debug token" token "along with x-waiter headers")
              (let [request-headers (merge service-description {:x-waiter-debug "true", "host" token})
                    path "/foo"
                    {:keys [headers request-headers] :as response} (make-request waiter-url path :headers request-headers)
                    service-id (retrieve-service-id waiter-url request-headers)]
                (assert-response-status response 200)
                (is (= (name-from-service-description waiter-url service-id) service-id-prefix))
                (is (= #{(make-source-tokens-entries waiter-url token)}
                       (service-id->source-tokens-entries waiter-url service-id)))
                (is (every? #(not (str/blank? (get headers %)))
                            (concat required-response-headers (retrieve-debug-response-headers waiter-url)))
                    (str headers))
                (is (= token-prefix (service-id->metric-group waiter-url service-id))
                    (str {:service-id service-id :token token}))
                (delete-service waiter-url service-id))))
          (finally
            (delete-token-and-assert waiter-url token)))))))

(deftest ^:parallel ^:integration-fast test-token-administer-unaffected-by-run-as-user-permissions
  (testing-using-waiter-url
    (let [service-id-prefix (rand-name)
          token-root (retrieve-token-root waiter-url)]
      (testing "token-administering"
        (testing "active-token"
          (let [last-update-time (System/currentTimeMillis)
                token (create-token-name waiter-url service-id-prefix)]

            (let [token-description {:health-check-url "/probe"
                                     :name service-id-prefix
                                     :owner (retrieve-username)
                                     :run-as-user (retrieve-username)
                                     :token token}
                  response (post-token waiter-url token-description)]
              (assert-response-status response 200))

            (testing "if-match-required during update"
              (let [token-description {:health-check-url "/probe-1"
                                       :last-update-time last-update-time
                                       :name service-id-prefix
                                       :owner (retrieve-username)
                                       :token token}
                    {:keys [body] :as response} (post-token waiter-url token-description
                                                            :headers {}
                                                            :query-params {"update-mode" "admin"})]
                (assert-response-status response 400)
                (is (str/includes? body "Must specify if-match header for admin mode token updates"))))

            (try
              (testing "successful admin mode update"
                (log/info "creating configuration using token" token)
                (let [{:keys [body headers]} (get-token waiter-url token)
                      token-description (-> body
                                            json/read-str
                                            walk/keywordize-keys
                                            (assoc :health-check-url "/probe-2"
                                                   :last-update-time last-update-time
                                                   :owner (retrieve-username)
                                                   :run-as-user "i-do-not-exist-but-will-not-be-checked"
                                                   :token token))
                      response (post-token waiter-url token-description
                                           :headers {"if-match" (get headers "etag")}
                                           :query-params {"update-mode" "admin"})]
                  (assert-response-status response 200)
                  (log/info "created configuration using token" token))

                (let [token-response (get-token waiter-url token)
                      response-body (json/read-str (:body token-response))]
                  (is (= {"health-check-url" "/probe-2"
                          "last-update-time" (-> last-update-time DateTime. du/date-to-str)
                          "last-update-user" (retrieve-username)
                          "name" service-id-prefix,
                          "owner" (retrieve-username)
                          "root" token-root
                          "run-as-user" "i-do-not-exist-but-will-not-be-checked"}
                         (dissoc response-body "previous")))
                  (log/info "asserted retrieval of configuration for token" token)))

              (testing "update with invalid etag"
                (let [token-description {:health-check-url "/probe-3"
                                         :last-update-time last-update-time
                                         :name service-id-prefix
                                         :owner (retrieve-username)
                                         :run-as-user "foo-bar"
                                         :token token}
                      {:keys [body] :as response} (post-token waiter-url token-description
                                                              :headers {"host" token
                                                                        "if-match" "1010"}
                                                              :query-params {"update-mode" "admin"})]
                  (assert-response-status response 412)
                  (is (str/includes? (str body) "Cannot modify stale token"))))

              (testing "hard-delete without etag"
                (let [{:keys [body] :as response} (make-request waiter-url "/token"
                                                                :headers {"host" token}
                                                                :method :delete
                                                                :query-params {"hard-delete" true})]
                  (assert-response-status response 400)
                  (is (str/includes? (str body) "Must specify if-match header for token hard deletes"))))

              (testing "hard-delete with invalid etag"
                (let [{:keys [body] :as response} (make-request waiter-url "/token"
                                                                :headers {"host" token
                                                                          "if-match" "1010"}
                                                                :method :delete
                                                                :query-params {"hard-delete" true})]
                  (assert-response-status response 412)
                  (is (str/includes? (str body) "Cannot modify stale token"))))

              (finally
                (delete-token-and-assert waiter-url token)))))

        (testing "deleted-token"
          (let [last-update-time (System/currentTimeMillis)
                token (create-token-name waiter-url service-id-prefix)]
            (try
              (log/info "creating configuration using token" token)
              (let [token-description {:health-check-url "/probe-3"
                                       :name service-id-prefix
                                       :owner (retrieve-username)
                                       :token token}
                    response (post-token waiter-url token-description :headers {"host" token})]
                (assert-response-status response 200))

              (let [{:keys [body headers]} (get-token waiter-url token)
                    token-description (-> body
                                          json/read-str
                                          walk/keywordize-keys
                                          (assoc :deleted true
                                                 :health-check-url "/probe"
                                                 :last-update-time last-update-time
                                                 :run-as-user "foo-bar"
                                                 :token token))
                    response (post-token waiter-url token-description
                                         :headers {"if-match" (get headers "etag")}
                                         :query-params {"update-mode" "admin"})]
                (assert-response-status response 200))
              (log/info "created configuration using token" token)
              (let [{:keys [body] :as response} (get-token waiter-url token)]
                (assert-response-status response 404)
                (is (str/includes? (str body) "Couldn't find token") (str body)))
              (let [token-response (get-token waiter-url token :query-params {"include" ["deleted" "metadata"]})
                    response-body (json/read-str (:body token-response))]
                (is (= {"deleted" true
                        "health-check-url" "/probe"
                        "last-update-time" (-> last-update-time DateTime. du/date-to-str)
                        "last-update-user" (retrieve-username)
                        "name" service-id-prefix
                        "owner" (retrieve-username)
                        "root" token-root
                        "run-as-user" "foo-bar"}
                       (dissoc response-body "previous"))))
              (log/info "asserted retrieval of configuration for token" token)
              (finally
                (delete-token-and-assert waiter-url token)))))))))

(deftest ^:parallel ^:integration-fast test-named-token
  (testing-using-waiter-url
    (log/info "basic named token test")
    (let [service-id-prefix (rand-name)
          token (create-token-name waiter-url service-id-prefix)]
      (try
        (log/info "creating configuration using token" token)
        (let [token-definition (assoc
                                 (kitchen-request-headers :prefix "")
                                 :name service-id-prefix
                                 :token token)
              {:keys [body status]} (post-token waiter-url token-definition)]
          (when (not= 200 status)
            (log/info "error:" body)
            (is (not body))))
        (log/info "created configuration using token" token)
        (log/info "retrieving configuration for token" token)
        (let [token-response (get-token waiter-url token)
              response-body (str (:body token-response))]
          (when (not (str/includes? response-body service-id-prefix))
            (log/info response-body))
          (assert-response-status token-response 200)
          (is (str/includes? response-body service-id-prefix)))
        (log/info "asserted retrieval of configuration for token" token)

        (log/info "making Waiter request with token" token "in header")
        (let [request-headers {:x-waiter-token token}
              path "/foo"
              response (make-request waiter-url path :headers request-headers)
              service-id (retrieve-service-id waiter-url (:request-headers response))]
          (assert-response-status response 200)
          (is (= (name-from-service-description waiter-url service-id) service-id-prefix))
          (is (= #{(make-source-tokens-entries waiter-url token)}
                 (service-id->source-tokens-entries waiter-url service-id))))

        (log/info "making Waiter request with token and x-waiter-debug token" token "in header")
        (let [request-headers {:x-waiter-debug "true", :x-waiter-token token}
              path "/foo"
              {:keys [headers request-headers] :as response} (make-request waiter-url path :headers request-headers)
              service-id (retrieve-service-id waiter-url request-headers)]
          (assert-response-status response 200)
          (is (= (name-from-service-description waiter-url service-id) service-id-prefix))
          (is (= #{(make-source-tokens-entries waiter-url token)}
                 (service-id->source-tokens-entries waiter-url service-id)))
          (is (every? #(not (str/blank? (get headers %)))
                      (concat required-response-headers (retrieve-debug-response-headers waiter-url)))
              (str headers))
          (delete-service waiter-url service-id))
        (finally
          (delete-token-and-assert waiter-url token))))))

(deftest ^:parallel ^:integration-fast test-star-run-as-user-token
  (testing-using-waiter-url
    (let [service-id-prefix (rand-name)
          token (create-token-name waiter-url service-id-prefix)]
      (try
        (log/info "creating configuration using token" token)
        (let [token-definition (assoc
                                 (kitchen-request-headers :prefix "")
                                 :name service-id-prefix
                                 :run-as-user "*"
                                 :token token)
              {:keys [body status]} (post-token waiter-url token-definition)]
          (when (not= 200 status)
            (log/info "error:" body)
            (is (not body))))
        (log/info "created configuration using token" token)
        (log/info "retrieving configuration for token" token)
        (let [token-response (get-token waiter-url token)
              response-body (str (:body token-response))]
          (when (not (str/includes? response-body service-id-prefix))
            (log/info response-body))
          (assert-response-status token-response 200)
          (is (str/includes? response-body service-id-prefix)))
        (log/info "asserted retrieval of configuration for token" token)

        (let [{:keys [body] :as token-response} (get-token waiter-url token)
              token-description (try (json/read-str (str body))
                                     (catch Exception _
                                       (is false (str "Failed to parse token" body))))]
          (assert-response-status token-response 200)
          (is (= "/status" (token-description "health-check-url")))
          (is (= service-id-prefix (token-description "name")))
          (is (= "*" (token-description "run-as-user")))
          (is (System/getProperty "user.name") (token-description "owner")))

        (log/info "making Waiter request with token" token "in header")
        (let [request-headers {:x-waiter-token token}
              path "/foo"
              response (make-request waiter-url path :headers request-headers)
              service-id (retrieve-service-id waiter-url (:request-headers response))]
          (assert-response-status response 200)
          (is (= (name-from-service-description waiter-url service-id) service-id-prefix))
          (is (= #{(make-source-tokens-entries waiter-url token)}
                 (service-id->source-tokens-entries waiter-url service-id)))
          (is (= (retrieve-username) (:run-as-user (service-id->service-description waiter-url service-id)))))

        (log/info "making Waiter request with token and x-waiter-debug token" token "in header")
        (let [request-headers {:x-waiter-debug "true", :x-waiter-token token}
              path "/foo"
              {:keys [headers request-headers] :as response} (make-request waiter-url path :headers request-headers)
              service-id (retrieve-service-id waiter-url request-headers)]
          (assert-response-status response 200)
          (is (= (name-from-service-description waiter-url service-id) service-id-prefix))
          (is (= #{(make-source-tokens-entries waiter-url token)}
                 (service-id->source-tokens-entries waiter-url service-id)))
          (is (every? #(not (str/blank? (get headers %)))
                      (concat required-response-headers (retrieve-debug-response-headers waiter-url)))
              (str headers))
          (delete-service waiter-url service-id))
        (finally
          (delete-token-and-assert waiter-url token))))))

(deftest ^:parallel ^:integration-fast ^:explicit test-on-the-fly-to-token
  (testing-using-waiter-url
    (let [name-string (rand-name)
          canary-response (make-kitchen-request waiter-url {:x-waiter-name name-string})
          service-id-1 (retrieve-service-id waiter-url (:request-headers canary-response))]
      (with-service-cleanup
        service-id-1
        (is (str/includes? service-id-1 name-string) (str "ERROR: App-name is missing " name-string))
        (is (nil? (service-id->source-tokens-entries waiter-url service-id-1)))
        (let [token (str "^SERVICE-ID#" service-id-1)
              response (make-request-with-debug-info {:x-waiter-token token} #(make-request waiter-url "" :headers %))
              service-id-2 (:service-id response)]
          (with-service-cleanup
            service-id-2
            (assert-response-status response 200)
            (is service-id-2)
            (is (= service-id-1 service-id-2) "The on-the-fly and token-based service ids do not match")
            (is (= #{(make-source-tokens-entries waiter-url token)}
                   (service-id->source-tokens-entries waiter-url service-id-2)))))))))

(deftest ^:parallel ^:integration-fast test-bad-token
  (testing-using-waiter-url

    (let [common-headers {"x-waiter-cmd" "foo-bar"
                          "x-waiter-cmd-type" "shell"}]
      (testing "ignore missing token when have service description"
        (let [response (make-request waiter-url "/pathabc" :headers (assoc common-headers "host" "missing_token"))]
          (is (str/includes? (:body response) "Service description using waiter headers/token improperly configured"))
          (assert-response-status response 400)))

      (testing "ignore invalid token when have service description"
        (let [response (make-request waiter-url "/pathabc" :headers (assoc common-headers "host" "bad/token"))]
          (is (str/includes? (:body response) "Service description using waiter headers/token improperly configured"))
          (assert-response-status response 400)))

      (testing "ignore invalid token when have invalid service description"
        (let [response (make-request waiter-url "/pathabc" :headers (assoc common-headers "host" "bad/token"))]
          (is (str/includes? (:body response) "Service description using waiter headers/token improperly configured"))
          (assert-response-status response 400))))
{'fraction_non_alphanumeric': 0.11786231048294084, 'fraction_numerical': 0.00590307631010548, 'mean_word_length': 1.4977085242896426, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 121, 'https://': 0, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 476, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6031015', 'n_tokens_mistral': 24472, 'n_tokens_neox': 22877, 'n_words': 6221}