<commit_before>(ns ^{:doc "provides search related utilities."}
  cmr.system-int-test.utils.search-util
  (:require [clojure.test :refer :all]
            [clj-http.client :as client]
            [clj-time.core :as t]
            [clj-time.coerce :as tc]
            [clojure.string :as str]
            [cheshire.core :as json]
            [cmr.common.concepts :as cs]
            [cmr.common.mime-types :as mime-types]
            [cmr.system-int-test.utils.url-helper :as url]
            [cmr.common.util :as util]
            [camel-snake-kebab :as csk]
            [clojure.set :as set]
            [clojure.walk]
            [ring.util.codec :as codec]
            [clojure.data.xml :as x]
            [cmr.common.xml :as cx]
            [cmr.umm.dif.collection]
            [cmr.umm.iso-mends.collection]
            [cmr.umm.iso-smap.collection]
            [cmr.system-int-test.data2.atom :as da]
            [cmr.system-int-test.data2.atom-json :as dj]
            [cmr.system-int-test.data2.provider-holdings :as ph]
            [cmr.system-int-test.data2.aql :as aql]
            [cmr.system-int-test.data2.aql-additional-attribute]
            [cmr.system-int-test.data2.facets :as f]))

(defn csv->tuples
  "Convert a comma-separated-value string into a set of tuples to be use with find-refs."
  [csv]
  (let [[type name min-value max-value] (str/split csv #"," -1)
        tuples [["attribute[][name]" name]
                ["attribute[][type]" type]]]
    (cond
      (and (not (empty? max-value)) (not (empty? min-value)))
      (into tuples [["attribute[][minValue]" min-value]
                    ["attribute[][maxValue]" max-value]])
      (not (empty? max-value))
      (conj tuples ["attribute[][maxValue]" max-value])

      max-value ;; max-value is empty but not nil
      (conj tuples ["attribute[][minValue]" min-value])

      :else ; min-value is really value
      (conj tuples ["attribute[][value]" min-value]))))

(defn params->snake_case
  "Converts search parameters to snake_case"
  [params]
  (->> params
       (util/map-keys
         (fn [k]
           (let [k (if (keyword? k) (name k) k)]
             (-> k
                 csk/->snake_case
                 (str/replace "_[" "[")
                 (str/replace "_]" "]")))))
       clojure.walk/keywordize-keys))

(deftest params->snake_case-test
  (is (= {(keyword "archive_center[]") ["SEDAC AC" "Larc" "Sedac AC"],
          (keyword "options[archive_center][and]") "false"
          :foo_bar "chew"}
         (params->snake_case
           {"archive-center[]" ["SEDAC AC" "Larc" "Sedac AC"],
            "options[archive-center][and]" "false"
            :foo-bar "chew"}))))

(defmacro get-search-failure-data
  "Executes a search and returns error data that was caught.
  Tests should verify the results this returns."
  [& body]
  `(try
     ~@body
     (catch clojure.lang.ExceptionInfo e#
       (let [{{status# :status body# :body} :object} (ex-data e#)
             errors# (:errors (json/decode body# true))]
         {:status status# :errors errors#}))))

(defmacro get-search-failure-xml-data
  "Executes a search and returns error data that was caught, parsing the body as an xml string.
  Tests should verify the results this returns."
  [& body]
  `(try
     ~@body
     (catch clojure.lang.ExceptionInfo e#
       (let [{{status# :status body# :body} :object} (ex-data e#)
             errors# (cx/strings-at-path (x/parse-str body#) [:error])]
         {:status status# :errors errors#}))))

(defn find-concepts-in-format
  "Returns the concepts in the format given."
  ([format concept-type params]
   (find-concepts-in-format format concept-type params {}))
  ([format concept-type params options]
   ;; no-snake-kebab needed for legacy psa which use camel case minValue/maxValue
   (let [url-extension (get options :url-extension)
         snake-kebab? (get options :snake-kebab? true)
         headers (get options :headers {})
         params (if snake-kebab?
                  (params->snake_case (util/map-keys csk/->snake_case_keyword params))
                  params)
         [url accept] (if url-extension
                        [(str (url/search-url concept-type) "." url-extension)]
                        [(url/search-url concept-type) format])
         response (client/get url {:accept accept
                                   :headers headers
                                   :query-params params<commit_msg>Added method for safely parsing the error response in XML.<commit_after>(ns ^{:doc "provides search related utilities."}
  cmr.system-int-test.utils.search-util
  (:require [clojure.test :refer :all]
            [clj-http.client :as client]
            [clj-time.core :as t]
            [clj-time.coerce :as tc]
            [clojure.string :as str]
            [cheshire.core :as json]
            [cmr.common.concepts :as cs]
            [cmr.common.mime-types :as mime-types]
            [cmr.system-int-test.utils.url-helper :as url]
            [cmr.common.util :as util]
            [camel-snake-kebab :as csk]
            [clojure.set :as set]
            [clojure.walk]
            [ring.util.codec :as codec]
            [clojure.data.xml :as x]
            [cmr.common.xml :as cx]
            [cmr.umm.dif.collection]
            [cmr.umm.iso-mends.collection]
            [cmr.umm.iso-smap.collection]
            [cmr.system-int-test.data2.atom :as da]
            [cmr.system-int-test.data2.atom-json :as dj]
            [cmr.system-int-test.data2.provider-holdings :as ph]
            [cmr.system-int-test.data2.aql :as aql]
            [cmr.system-int-test.data2.aql-additional-attribute]
            [cmr.system-int-test.data2.facets :as f]))

(defn csv->tuples
  "Convert a comma-separated-value string into a set of tuples to be use with find-refs."
  [csv]
  (let [[type name min-value max-value] (str/split csv #"," -1)
        tuples [["attribute[][name]" name]
                ["attribute[][type]" type]]]
    (cond
      (and (not (empty? max-value)) (not (empty? min-value)))
      (into tuples [["attribute[][minValue]" min-value]
                    ["attribute[][maxValue]" max-value]])
      (not (empty? max-value))
      (conj tuples ["attribute[][maxValue]" max-value])

      max-value ;; max-value is empty but not nil
      (conj tuples ["attribute[][minValue]" min-value])

      :else ; min-value is really value
      (conj tuples ["attribute[][value]" min-value]))))

(defn params->snake_case
  "Converts search parameters to snake_case"
  [params]
  (->> params
       (util/map-keys
         (fn [k]
           (let [k (if (keyword? k) (name k) k)]
             (-> k
                 csk/->snake_case
                 (str/replace "_[" "[")
                 (str/replace "_]" "]")))))
       clojure.walk/keywordize-keys))

(deftest params->snake_case-test
  (is (= {(keyword "archive_center[]") ["SEDAC AC" "Larc" "Sedac AC"],
          (keyword "options[archive_center][and]") "false"
          :foo_bar "chew"}
         (params->snake_case
           {"archive-center[]" ["SEDAC AC" "Larc" "Sedac AC"],
            "options[archive-center][and]" "false"
            :foo-bar "chew"}))))

(defmacro get-search-failure-data
  "Executes a search and returns error data that was caught.
  Tests should verify the results this returns."
  [& body]
  `(try
     ~@body
     (catch clojure.lang.ExceptionInfo e#
       (let [{{status# :status body# :body} :object} (ex-data e#)
             errors# (:errors (json/decode body# true))]
         {:status status# :errors errors#}))))

(defn safe-parse-error-xml
  [xml]
  (try
    (cx/strings-at-path (x/parse-str xml) [:error])
    (catch Exception e
      (.printStackTrace e)
      [xml])))

(defmacro get-search-failure-xml-data
  "Executes a search and returns error data that was caught, parsing the body as an xml string.
  Tests should verify the results this returns."
  [& body]
  `(try
     ~@body
     (catch clojure.lang.ExceptionInfo e#
       (let [{{status# :status body# :body} :object} (ex-data e#)
             errors# (safe-parse-error-xml body#)]
         {:status status# :errors errors#}))))

(defn find-concepts-in-format
  "Returns the concepts in the format given."
  ([format concept-type params]
   (find-concepts-in-format format concept-type params {}))
  ([format concept-type params options]
   ;; no-snake-kebab needed for legacy psa which use camel case minValue/maxValue
   (let [url-extension (get options :url-extension)
         snake-kebab? (get options :snake-kebab? true)
         headers (get options :headers {})
         params (if snake-kebab?
                  (params->snake_case (util/map-keys csk/->snake_case_keyword params))
                  params)
         [url accept] (if url-extension
                        [(str (url/search-url concept-type) "." url-extension)]
                        [(url/search-url concept-type) format])
         response (client/get url {:accept accept
                                   :headers headers
                                   :query-params params
{'fraction_non_alphanumeric': 0.1439911797133407, 'fraction_numerical': 0.0015435501653803748, 'mean_word_length': 2.105443341321465, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 13}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4255108', 'n_tokens_mistral': 2984, 'n_tokens_neox': 2889, 'n_words': 736}