<commit_before>(ns cmr.metadata-db.int-test.utility
  "Contains various utitiltiy methods to support integeration tests."
  (:require [clojure.test :refer :all]
            [clj-http.client :as client]
            [cheshire.core :as cheshire]))

;;; Enpoints for services - change this for tcp-mon
(def port 3001)

(def concepts-url (str "http://localhost:" port "/concepts/"))

(def concept-id-url (str "http://localhost:" port "/concept-id/"))

(def reset-url (str "http://localhost:" port "/reset"))


;;; utility methods
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn concept
  "Creates a concept to be used for testing."
  []
  {
   :concept-id ""
   :concept-type :collection
   :native-id "provider collection id"
   :provider-id "PROV1"
   :metadata "xml here"
   :format "echo10"})

(defn get-concept-id
  "Make a GET to retrieve the id for a given concept-type, provider-id, and native-id."
  [concept-type provider-id native-id]
  (let [response (client/get (str concept-id-url concept-type "/" provider-id "/" native-id)
                             {:accept :json
                              :throw-exceptions false})
        status (:status response)
        body (cheshire/parse-string (:body response))
        concept-id (get body  "concept-id")
        errors (get body "errors")]
    {:status status :concept-id concept-id :error-messages errors}))

(defn get-concept-by-id-and-revision
  "Make a GET to retrieve a concept by concept-id and revision."
  [concept-id revision-id]
  (let [response (client/get (str concepts-url concept-id "/" revision-id)
                             {:accept :json
                              :throw-exceptions false})
        status (:status response)]
    (if (= status 200)
      (let [found-concept (clojure.walk/keywordize-keys (cheshire/parse-string (:body response)))]
        {:status status :concept found-concept})
      {:status status :concept nil})))

(defn get-concept-by-id
  "Make a GET to retrieve a concept by concept-id."
  [concept-id]
  (let [response (client/get (str concepts-url concept-id)
                             {:accept :json
                              :throw-exceptions false})
        status (:status response)]
    (if (= status 200)
      (let [found-concept (clojure.walk/keywordize-keys (cheshire/parse-string (:body response)))]
        {:status status :concept found-concept})
      {:status status :concept nil})))

(defn get-concepts
  "Make a POST to retrieve concepts by concept-id and revision."
  [tuples]
  (let [body {:concept-revisions tuples}]
    (let [response (client/post (str concepts-url "search")
                                {:body (cheshire/generate-string body)
                                 :body-encoding "UTF-8"
                                 :content-type :json
                                 :accept :json
                                 :throw-exceptions false})
          status (:status response)
          concepts (vec (cheshire/parse-string (:body response)))]
      {:status status :concepts concepts})))

(defn concepts-and-concept-id-revisions-equal?
  "Compare a vector of concepts returned by the API to a set of concept-id/revision-ids."
  [concepts concept-id-revision-ids]
  (if (not= (count concepts) (count concept-id-revision-ids))
    false
    (every? true? 
            (map (fn [[con-id rev-id]] 
                   (some (fn [concept]
                           (and (= (get concept "concept-id") con-id) 
                                (= (get concept "revision-id" rev-id))))
                         concepts))
                 concept-id-revision-ids))))

(defn concepts-and-ids-equal?
  ;; TODO - this might need to be changed to be order independent
    "Compare a vector of concepts returned by the API to a set of concept-ids"
    [concepts concept-ids]
    (if (not= (count concepts) (count concept-ids))
      false
      (every? true? (map #(= (get %1 "concept-id") %2) concepts concept-ids))))

(defn save-concept
  "Make a POST request to save a concept with JSON encoding of the concept.  Returns a map with
  status, revision-id, and a list of error messages"
  [concept]
  (let [response (client/post concepts-url
                              {:body (cheshire/generate-string concept)
                               :body-encoding "UTF-8"
                               :content-type :json
                               :accept :json
                               :throw-exceptions false})
        status (:status response)
        body (cheshire/parse-string (:body response))
        revision-id (get body "revision-id")
        concept-id (get body "concept-id")
        error-messages (get body "errors")]
    {:status status :revision-id revision-id :concept-id concept-id :error-messages error-messages}))

(defn delete-concept
  "Make a DELETE request to mark a concept as deleted. Returns the status and revision id of the
  tombstone."
  [concept-id & revision-id]
  (let [revision-id (first revision-id)
        url (if revision-id 
              (format "%s%s/%s" concepts-url concept-id revision-id)
              (format "%s%s" concepts-url concept-id))
        response (client/delete url
                                {:throw-exceptions false})
        status (:status response)
        body (cheshire/parse-string (:body response))
        revision-id (get body "revision-id")
        error-messages (get body "errors")]
    {:status status :revision-id revision-id :error-messages error-messages}))

(defn force-delete-concept
  "Make a DELETE request to permanently remove a revison of a concept."
  [concept-id revision-id]
  (let [url (format "%sforce-delete/%s/%s" concepts-url concept-id revision-id)
        response (client/delete url<commit_msg>Removing a temporary change in the metadata db test.<commit_after>(ns cmr.metadata-db.int-test.utility
  "Contains various utitiltiy methods to support integeration tests."
  (:require [clojure.test :refer :all]
            [clj-http.client :as client]
            [cheshire.core :as cheshire]))

;;; Enpoints for services - change this for tcp-mon
(def port 3001)

(def concepts-url (str "http://localhost:" port "/concepts/"))

(def concept-id-url (str "http://localhost:" port "/concept-id/"))

(def reset-url (str "http://localhost:" port "/reset"))


;;; utility methods
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(defn concept
  "Creates a concept to be used for testing."
  []
  {:concept-type :collection
   :native-id "provider collection id"
   :provider-id "PROV1"
   :metadata "xml here"
   :format "echo10"})

(defn get-concept-id
  "Make a GET to retrieve the id for a given concept-type, provider-id, and native-id."
  [concept-type provider-id native-id]
  (let [response (client/get (str concept-id-url concept-type "/" provider-id "/" native-id)
                             {:accept :json
                              :throw-exceptions false})
        status (:status response)
        body (cheshire/parse-string (:body response))
        concept-id (get body  "concept-id")
        errors (get body "errors")]
    {:status status :concept-id concept-id :error-messages errors}))

(defn get-concept-by-id-and-revision
  "Make a GET to retrieve a concept by concept-id and revision."
  [concept-id revision-id]
  (let [response (client/get (str concepts-url concept-id "/" revision-id)
                             {:accept :json
                              :throw-exceptions false})
        status (:status response)]
    (if (= status 200)
      (let [found-concept (clojure.walk/keywordize-keys (cheshire/parse-string (:body response)))]
        {:status status :concept found-concept})
      {:status status :concept nil})))

(defn get-concept-by-id
  "Make a GET to retrieve a concept by concept-id."
  [concept-id]
  (let [response (client/get (str concepts-url concept-id)
                             {:accept :json
                              :throw-exceptions false})
        status (:status response)]
    (if (= status 200)
      (let [found-concept (clojure.walk/keywordize-keys (cheshire/parse-string (:body response)))]
        {:status status :concept found-concept})
      {:status status :concept nil})))

(defn get-concepts
  "Make a POST to retrieve concepts by concept-id and revision."
  [tuples]
  (let [body {:concept-revisions tuples}]
    (let [response (client/post (str concepts-url "search")
                                {:body (cheshire/generate-string body)
                                 :body-encoding "UTF-8"
                                 :content-type :json
                                 :accept :json
                                 :throw-exceptions false})
          status (:status response)
          concepts (vec (cheshire/parse-string (:body response)))]
      {:status status :concepts concepts})))

(defn concepts-and-concept-id-revisions-equal?
  "Compare a vector of concepts returned by the API to a set of concept-id/revision-ids."
  [concepts concept-id-revision-ids]
  (if (not= (count concepts) (count concept-id-revision-ids))
    false
    (every? true?
            (map (fn [[con-id rev-id]]
                   (some (fn [concept]
                           (and (= (get concept "concept-id") con-id)
                                (= (get concept "revision-id" rev-id))))
                         concepts))
                 concept-id-revision-ids))))

(defn concepts-and-ids-equal?
  ;; TODO - this might need to be changed to be order independent
    "Compare a vector of concepts returned by the API to a set of concept-ids"
    [concepts concept-ids]
    (if (not= (count concepts) (count concept-ids))
      false
      (every? true? (map #(= (get %1 "concept-id") %2) concepts concept-ids))))

(defn save-concept
  "Make a POST request to save a concept with JSON encoding of the concept.  Returns a map with
  status, revision-id, and a list of error messages"
  [concept]
  (let [response (client/post concepts-url
                              {:body (cheshire/generate-string concept)
                               :body-encoding "UTF-8"
                               :content-type :json
                               :accept :json
                               :throw-exceptions false})
        status (:status response)
        body (cheshire/parse-string (:body response))
        revision-id (get body "revision-id")
        concept-id (get body "concept-id")
        error-messages (get body "errors")]
    {:status status :revision-id revision-id :concept-id concept-id :error-messages error-messages}))

(defn delete-concept
  "Make a DELETE request to mark a concept as deleted. Returns the status and revision id of the
  tombstone."
  [concept-id & revision-id]
  (let [revision-id (first revision-id)
        url (if revision-id
              (format "%s%s/%s" concepts-url concept-id revision-id)
              (format "%s%s" concepts-url concept-id))
        response (client/delete url
                                {:throw-exceptions false})
        status (:status response)
        body (cheshire/parse-string (:body response))
        revision-id (get body "revision-id")
        error-messages (get body "errors")]
    {:status status :revision-id revision-id :error-messages error-messages}))

(defn force-delete-concept
  "Make a DELETE request to permanently remove a revison of a concept."
  [concept-id revision-id]
  (let [url (format "%sforce-delete/%s/%s" concepts-url concept-id revision-id)
        response (client/delete url
{'fraction_non_alphanumeric': 0.12596579564198282, 'fraction_numerical': 0.002951645108082299, 'mean_word_length': 2.1788079470198674, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3304506', 'n_tokens_mistral': 3390, 'n_tokens_neox': 3226, 'n_words': 1030}