<commit_before>(ns cmr.system-int-test.search.collection-concept-retrieval-test
  "Integration test for collection retrieval via the /concepts/:concept-id and
  /concepts/:concept-id/:revision-id endpoints."
  (:require
   [cheshire.core :as json]
   [clj-http.client :as client]
   [clj-time.format :as f]
   [clojure.data.xml :as x]
   [clojure.string :as str]
   [clojure.test :refer :all]
   [cmr.common.mime-types :as mt]
   [cmr.common.mime-types :as mt]
   [cmr.common.util :refer [are3] :as util]
   [cmr.common.xml :as cx]
   [cmr.mock-echo.client.echo-util :as e]
   [cmr.spatial.codec :as codec]
   [cmr.spatial.line-string :as l]
   [cmr.spatial.mbr :as m]
   [cmr.spatial.point :as p]
   [cmr.spatial.polygon :as poly]
   [cmr.spatial.ring-relations :as rr]
   [cmr.system-int-test.data2.atom :as da]
   [cmr.system-int-test.data2.atom-json :as dj]
   [cmr.system-int-test.data2.collection :as dc]
   [cmr.system-int-test.data2.core :as d]
   [cmr.system-int-test.data2.granule :as dg]
   [cmr.system-int-test.data2.kml :as dk]
   [cmr.system-int-test.system :as s]
   [cmr.system-int-test.utils.fast-xml :as fx]
   [cmr.system-int-test.utils.index-util :as index]
   [cmr.system-int-test.utils.ingest-util :as ingest]
   [cmr.system-int-test.utils.search-util :as search]
   [cmr.system-int-test.utils.url-helper :as url]
   [cmr.transmit.config :as transmit-config]
   [cmr.umm-spec.test.expected-conversion :as expected-conversion]
   [cmr.umm-spec.test.location-keywords-helper :as lkt]
   [cmr.umm-spec.umm-json :as umm-json]
   [cmr.umm-spec.umm-spec-core :as umm-spec]
   [cmr.umm-spec.versioning :as ver]
   [cmr.umm.echo10.echo10-collection :as c]
   [cmr.umm.iso-mends.iso-mends-collection :as umm-c]
   [cmr.umm.umm-core :as umm]
   [cmr.umm.umm-spatial :as umm-s]))

(use-fixtures
  :each
  (ingest/reset-fixture
    {"provguid1" "PROV1"
     "provguid2" "PROV2"
     "provguid3" "PROV3"
     "provguid4" "PROV4"
     "usgsguid"  "USGS_EROS"}
    {:grant-all-search? false}))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;
;; Utility functions
;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(def test-context (lkt/setup-context-for-test))

(defn- get-concept-by-id-helper
  [concept options]
  (:body (search/retrieve-concept (:concept-id concept) nil options)))

(defmulti result-matches?
  "Compare UMM record to the response from concept retrieval"
  (fn [original-format format-key umm response]
    format-key))

(defn- update-iso-entry-title
  "Returns the ISO19115 collection with entry title updated to the given value.
  This is a temporary workaround and should be removed once CMR-3256 is fixed."
  [coll correct-entry-title]
  (assoc coll :entry-title correct-entry-title))

;; ISO-19115 must be handled separately from the other formats becaue it uses xslt to translate
;; ECHO10 and the resulting XML is not quite the same as what we get when going from UMM to XML.
(defmethod result-matches? :iso19115
  [original-format format-key umm response]
  (let [metadata-xml (:body response)
        metadata-umm (-> (umm-c/parse-collection metadata-xml)
                         ;; Update entry-title to the correct value as the xslt is not generating
                         ;; the correct entry-title as of now.
                         ;; This is temporary and should be removed once CMR-3256 is fixed.
                         (update-iso-entry-title (:entry-title umm))
                         ;; remove default added by parser
                         (assoc :metadata-language nil)
                         ;; remove default added by parser
                         (assoc :use-constraints nil))]
    (is (= umm metadata-umm))))

(defmethod result-matches? :default
  [original-format format-key umm response]
  (let [expected (d/expected-metadata test-context :collection original-format format-key umm)
        metadata-xml (:body response)]
    (if (= :iso-smap format-key)
      (is (= (expected-conversion/ignore-ids expected)
             (expected-conversion/ignore-ids metadata-xml)))
      (is (= expected metadata-xml)))))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;
;; Retrieve by concept-id - general test
;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(deftest retrieve-collection-by-cmr-concept-id

  ;; Ingest 2 early versions of coll1
  (d/ingest "PROV1" (dc/collection {:entry-title "coll1"
                                    :projects (dc/projects "ESI_1")}))
  (d/ingest "PROV1" (dc/collection {:entry-title "coll1"
                                    :projects (dc/projects "ESI_2")}))

  (let [umm-coll (dc/collection {:entry-title "coll1"
                                 :projects (dc/projects "ESI_3")})
        coll1 (d/ingest "PROV1" umm-coll)
        coll2 (d/ingest "PROV1" (dc/collection {:entry-title "Dataset2"}))
        del-coll (d/ingest "PROV1" (dc/collection))
        ;; tokens
        guest-token (e/login-guest (s/context))
        user1-token (e/login (s/context) "user1")]
    (ingest/delete-concept (d/item->concept del-coll :echo10))
    (index/wait-until-indexed)

    ;; Registered users have access to coll1
    (e/grant-registered-users (s/context) (e/coll-catalog-item-id "PROV1" (e/coll-id ["coll1"])))
    (ingest/reindex-collection-permitted-groups (transmit-config/echo-system-token))
    (index/wait-until-indexed)

    (testing "retrieval of a deleted collection results in a 404"
      (let [{:keys [status errors]} (search/get-search-failure-xml-data
                                      (search/retrieve-concept
                                        (:concept-id del-coll) nil {:throw-exceptions true
                                                                    :headers {transmit-config/token-header
                                                                              user1-token}}))]
        (is (= 404 status))
        (is (= ["Concept with concept-id [C1200000018-PROV1] could not be found."] errors))))
    (testing "retrieval by collection cmr-concept-id returns the latest revision."
      (let [response (search/retrieve-concept
                       (:concept-id coll1) nil {:query-params {:token user1-token}})
            parsed-collection (c/parse-collection (:body response))]
        (is (search/mime-type-matches-response? response mt/echo10))
        (is (= umm-coll parsed-collection))))
    (testing "retrieval with .xml extension returns correct mime type"
      (let [response (search/retrieve-concept
                       (:concept-id coll1) nil {:query-params {:token user1-token}
                                                :url-extension "xml"})
            parsed-collection (c/parse-collection (:body response))]
        (is (search/mime-type-matches-response? response mt/echo10))
        (is (= umm-coll parsed-collection))))
    (testing "retrieval by collection cmr-concept-id, not found."
      (let [{:keys [status errors]} (search/get-search-failure-xml-data
                                      (search/retrieve-concept
                                        "C1111-PROV1"  nil {:throw-exceptions true
                                                            :headers {transmit-config/token-header
                                                                      user1-token}}))]
        (is (= 404 status))
        (is (= ["Concept with concept-id [C1111-PROV1] could not be found."] errors))))
    (testing "retrieval of HTML with extension"
      (let [response (search/retrieve-concept
                       (:concept-id coll1) nil {:query-params {:token user1-token}
                                                :url-extension "html"})
            _ (is (= 200 (:status response)))]
        (is (search/mime-type-matches-response? response mt/html))
        (is (= "C1200000016-PROV1" (:concept-id coll1) ))
        (is (.contains ^String (:body response) (:entry-title umm-coll)))))
    (testing "retrieval of HTML with accept headers"
      (let [response (search/retrieve-concept
                       (:concept-id coll1) nil {:query-params {:token user1-token}
                                                :accept "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"})
            _ (is (= 200 (:status response)))]
        (is (search/mime-type-matches-response? response mt/html))
        (is (.contains ^String (:body response) (:entry-title umm-coll)))))
    (testing "retrieval of UMM JSON"
      (let [response (search/retrieve-concept
                       (:concept-id coll1) nil {:query-params {:token user1-token}
                                                :url-extension "umm-json"})
            _ (is (= 200 (:status response)))
            parsed-collection (umm-json/json->umm test-context :collection (:body response))]
        (is (search/mime-type-matches-response? response mt/umm-json))
        (is (= (:entry-title umm-coll) (:EntryTitle parsed-collection)))))))

(defn- expected-umm-json
  "Returns the expected umm json in the expected-version for the given metadata whose umm
  json version is source-version."<commit_msg>CMR-4703 - changes from PR review<commit_after>(ns cmr.system-int-test.search.collection-concept-retrieval-test
  "Integration test for collection retrieval via the /concepts/:concept-id and
  /concepts/:concept-id/:revision-id endpoints."
  (:require
   [cheshire.core :as json]
   [clj-http.client :as client]
   [clj-time.format :as f]
   [clojure.data.xml :as x]
   [clojure.string :as str]
   [clojure.test :refer :all]
   [cmr.common.mime-types :as mt]
   [cmr.common.mime-types :as mt]
   [cmr.common.util :refer [are3] :as util]
   [cmr.common.xml :as cx]
   [cmr.mock-echo.client.echo-util :as e]
   [cmr.spatial.codec :as codec]
   [cmr.spatial.line-string :as l]
   [cmr.spatial.mbr :as m]
   [cmr.spatial.point :as p]
   [cmr.spatial.polygon :as poly]
   [cmr.spatial.ring-relations :as rr]
   [cmr.system-int-test.data2.atom :as da]
   [cmr.system-int-test.data2.atom-json :as dj]
   [cmr.system-int-test.data2.collection :as dc]
   [cmr.system-int-test.data2.core :as d]
   [cmr.system-int-test.data2.granule :as dg]
   [cmr.system-int-test.data2.kml :as dk]
   [cmr.system-int-test.system :as s]
   [cmr.system-int-test.utils.fast-xml :as fx]
   [cmr.system-int-test.utils.index-util :as index]
   [cmr.system-int-test.utils.ingest-util :as ingest]
   [cmr.system-int-test.utils.search-util :as search]
   [cmr.system-int-test.utils.url-helper :as url]
   [cmr.transmit.config :as transmit-config]
   [cmr.umm-spec.test.expected-conversion :as expected-conversion]
   [cmr.umm-spec.test.location-keywords-helper :as lkt]
   [cmr.umm-spec.umm-json :as umm-json]
   [cmr.umm-spec.umm-spec-core :as umm-spec]
   [cmr.umm-spec.versioning :as ver]
   [cmr.umm.echo10.echo10-collection :as c]
   [cmr.umm.iso-mends.iso-mends-collection :as umm-c]
   [cmr.umm.umm-core :as umm]
   [cmr.umm.umm-spatial :as umm-s]))

(use-fixtures
  :each
  (ingest/reset-fixture
    {"provguid1" "PROV1"
     "provguid2" "PROV2"
     "provguid3" "PROV3"
     "provguid4" "PROV4"
     "usgsguid"  "USGS_EROS"}
    {:grant-all-search? false}))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;
;; Utility functions
;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(def test-context (lkt/setup-context-for-test))

(defn- get-concept-by-id-helper
  [concept options]
  (:body (search/retrieve-concept (:concept-id concept) nil options)))

(defmulti result-matches?
  "Compare UMM record to the response from concept retrieval"
  (fn [original-format format-key umm response]
    format-key))

(defn- update-iso-entry-title
  "Returns the ISO19115 collection with entry title updated to the given value.
  This is a temporary workaround and should be removed once CMR-3256 is fixed."
  [coll correct-entry-title]
  (assoc coll :entry-title correct-entry-title))

;; ISO-19115 must be handled separately from the other formats becaue it uses xslt to translate
;; ECHO10 and the resulting XML is not quite the same as what we get when going from UMM to XML.
(defmethod result-matches? :iso19115
  [original-format format-key umm response]
  (let [metadata-xml (:body response)
        metadata-umm (-> (umm-c/parse-collection metadata-xml)
                         ;; Update entry-title to the correct value as the xslt is not generating
                         ;; the correct entry-title as of now.
                         ;; This is temporary and should be removed once CMR-3256 is fixed.
                         (update-iso-entry-title (:entry-title umm))
                         ;; remove default added by parser
                         (assoc :metadata-language nil)
                         ;; remove default added by parser
                         (assoc :use-constraints nil))]
    (is (= umm metadata-umm))))

(defmethod result-matches? :default
  [original-format format-key umm response]
  (let [expected (d/expected-metadata test-context :collection original-format format-key umm)
        metadata-xml (:body response)]
    (if (= :iso-smap format-key)
      (is (= (expected-conversion/ignore-ids expected)
             (expected-conversion/ignore-ids metadata-xml)))
      (is (= expected metadata-xml)))))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;
;; Retrieve by concept-id - general test
;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(deftest retrieve-collection-by-cmr-concept-id

  ;; Ingest 2 early versions of coll1
  (d/ingest "PROV1" (dc/collection {:entry-title "coll1"
                                    :projects (dc/projects "ESI_1")}))
  (d/ingest "PROV1" (dc/collection {:entry-title "coll1"
                                    :projects (dc/projects "ESI_2")}))

  (let [umm-coll (dc/collection {:entry-title "coll1"
                                 :projects (dc/projects "ESI_3")})
        coll1 (d/ingest "PROV1" umm-coll)
        coll2 (d/ingest "PROV1" (dc/collection {:entry-title "Dataset2"}))
        del-coll (d/ingest "PROV1" (dc/collection))
        ;; tokens
        guest-token (e/login-guest (s/context))
        user1-token (e/login (s/context) "user1")]
    (ingest/delete-concept (d/item->concept del-coll :echo10))
    (index/wait-until-indexed)

    ;; Registered users have access to coll1
    (e/grant-registered-users (s/context) (e/coll-catalog-item-id "PROV1" (e/coll-id ["coll1"])))
    (ingest/reindex-collection-permitted-groups (transmit-config/echo-system-token))
    (index/wait-until-indexed)

    (testing "retrieval of a deleted collection results in a 404"
      (let [{:keys [status errors]} (search/get-search-failure-xml-data
                                      (search/retrieve-concept
                                        (:concept-id del-coll) nil {:throw-exceptions true
                                                                    :headers {transmit-config/token-header
                                                                              user1-token}}))]
        (is (= 404 status))
        (is (= ["Concept with concept-id [C1200000018-PROV1] could not be found."] errors))))
    (testing "retrieval by collection cmr-concept-id returns the latest revision."
      (let [response (search/retrieve-concept
                       (:concept-id coll1) nil {:query-params {:token user1-token}})
            parsed-collection (c/parse-collection (:body response))]
        (is (search/mime-type-matches-response? response mt/echo10))
        (is (= umm-coll parsed-collection))))
    (testing "retrieval with .xml extension returns correct mime type"
      (let [response (search/retrieve-concept
                       (:concept-id coll1) nil {:query-params {:token user1-token}
                                                :url-extension "xml"})
            parsed-collection (c/parse-collection (:body response))]
        (is (search/mime-type-matches-response? response mt/echo10))
        (is (= umm-coll parsed-collection))))
    (testing "retrieval by collection cmr-concept-id, not found."
      (let [{:keys [status errors]} (search/get-search-failure-xml-data
                                      (search/retrieve-concept
                                        "C1111-PROV1"  nil {:throw-exceptions true
                                                            :headers {transmit-config/token-header
                                                                      user1-token}}))]
        (is (= 404 status))
        (is (= ["Concept with concept-id [C1111-PROV1] could not be found."] errors))))
    (testing "retrieval of HTML with extension"
      (let [response (search/retrieve-concept
                       (:concept-id coll1) nil {:query-params {:token user1-token}
                                                :url-extension "html"})
            _ (is (= 200 (:status response)))]
        (is (search/mime-type-matches-response? response mt/html))
        (is (.contains ^String (:body response) (:concept-id coll1)))
        (is (.contains ^String (:body response) (:entry-title umm-coll)))))
    (testing "retrieval of HTML with accept headers"
      (let [response (search/retrieve-concept
                       (:concept-id coll1) nil {:query-params {:token user1-token}
                                                :accept "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"})
            _ (is (= 200 (:status response)))]
        (is (search/mime-type-matches-response? response mt/html))
        (is (.contains ^String (:body response) (:entry-title umm-coll)))))
    (testing "retrieval of UMM JSON"
      (let [response (search/retrieve-concept
                       (:concept-id coll1) nil {:query-params {:token user1-token}
                                                :url-extension "umm-json"})
            _ (is (= 200 (:status response)))
            parsed-collection (umm-json/json->umm test-context :collection (:body response))]
        (is (search/mime-type-matches-response? response mt/umm-json))
        (is (= (:entry-title umm-coll) (:EntryTitle parsed-collection)))))))

(defn- expected-umm-json
  "Returns the expected umm json in the expected-version for the given metadata whose umm
  json version is source-version."
{'fraction_non_alphanumeric': 0.16927950177171697, 'fraction_numerical': 0.01444217760120262, 'mean_word_length': 2.328032874754333, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 28}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 50, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4048875', 'n_tokens_mistral': 6208, 'n_tokens_neox': 5617, 'n_words': 1356}