<commit_before>(ns cmr.system-int-test.ingest.granule-validation-test
  "CMR Ingest granule validation integration tests"
  (:require
    [clj-time.core :as t]
    [clj-time.format :as tf ]
    [clojure.java.io :as io]
    [clojure.java.shell :as shell]
    [clojure.test :refer :all]
    [cmr.common.time-keeper :as tk]
    [cmr.common.util :refer [are2]]
    [cmr.ingest.services.messages :as msg]
    [cmr.spatial.line-string :as l]
    [cmr.spatial.mbr :as m]
    [cmr.spatial.point :as p]
    [cmr.spatial.polygon :as poly]
    [cmr.system-int-test.data2.core :as d]
    [cmr.system-int-test.data2.granule :as dg]
    [cmr.system-int-test.data2.umm-spec-collection :as data-umm-c]
    [cmr.system-int-test.data2.umm-spec-common :as data-umm-cmn]
    [cmr.system-int-test.utils.dev-system-util :as dev-sys-util]
    [cmr.system-int-test.utils.ingest-util :as ingest]
    [cmr.system-int-test.utils.url-helper :as url]
    [cmr.umm.umm-granule :as umm-g]
    [cmr.umm.umm-spatial :as umm-s]))

(use-fixtures :each (ingest/reset-fixture {"provguid1" "PROV1" "provguid2" "PROV2"}))

(comment
  (do
    (dev-sys-util/reset)
    (ingest/create-provider {:provider-guid "provguid1" :provider-id "PROV1"}))

  (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection
                      {:AdditionalAttributes
                       [(data-umm-cmn/additional-attribute {:Name "bool" :DataType :boolean :value true})
                        (data-umm-cmn/additional-attribute {:Name "bool" :DataType :boolean :Value true})]})))



(defn assert-validation-errors
  "Asserts that when the granule and optional collection concept are validated the expected errors
  are returned. The collection concept can be passed as a third argument and it will be sent along
  with the granule instead of using a previously ingested collection."
  [expected-status-code expected-errors & gran-and-optional-coll-concept]
  (let [response (apply ingest/validate-granule gran-and-optional-coll-concept)]
    (is (= {:status expected-status-code :errors expected-errors}
           (select-keys response [:status :errors])))))

(defn assert-validation-success
  "Asserts that when the granule and optional collection concept are valid. The collection concept
  can be passed as a third argument and it will be sent along with the granule instead of using a
  previously ingested collection."
  [& gran-and-optional-coll-concept]
  (let [response (apply ingest/validate-granule gran-and-optional-coll-concept)]
    (is (= {:status 200} (select-keys response [:status :errors])))))

(deftest validation-endpoint-test
  (let [invalid-granule-xml "<Granule>invalid xml</Granule>"
        expected-errors ["Line 1 - cvc-complex-type.2.3: Element 'Granule' cannot have character [children], because the type's content type is element-only."
                         "Line 1 - cvc-complex-type.2.4.b: The content of element 'Granule' is not complete. One of '{GranuleUR}' is expected."]]

    (testing "granule with end date slightly in the future and collection with
             EndsAtPresentFlag=true and end date in the far future (see CMR-1351)"
             (let [collection (data-umm-c/collection {:EntryTitle "correct"
                                              :ShortName "S1"
                                              :Version "V1"
                                              :TemporalExtents [(data-umm-cmn/temporal-extent
                                                          {:beginning-date-time "2009-03-08T00:00:00Z"
                                                           :ending-date-time "2121-11-04T00:00:00Z"
                                                           :ends-at-present? true})]})
                   coll-concept (d/umm-c-collection->concept collection)
                   future-date-time (t/plus (tk/now) (t/hours 1))
                   time-formatter (tf/formatters :date-time-no-ms)
                   future-date-time-str (tf/unparse time-formatter future-date-time)
                   granule (assoc (dg/granule-with-umm-spec-collection collection (:concept-id collection))
                                  :temporal (dg/temporal {:beginning-date-time "2010-11-04T00:00:00Z"
                                                          :ending-date-time future-date-time-str}))]
               (assert-validation-success
                 (d/item->concept granule)
                 coll-concept)))

    (testing "with collection as additional parameter"
      (let [collection (data-umm-c/collection {})
            coll-concept (d/umm-c-collection->concept collection :echo10)]
        (testing "success"
          (let [concept (d/item->concept (dg/granule-with-umm-spec-collection collection (:concept-id collection)))]
            (assert-validation-success concept coll-concept)))
        (testing "collection in different format than granule"
          (let [concept (d/item->concept (dg/granule-with-umm-spec-collection collection (:concept-id collection)))]
            (assert-validation-success concept (d/umm-c-collection->concept collection :iso19115))
            (assert-validation-success concept (d/umm-c-collection->concept collection :dif))))

        (testing "invalid collection xml"
          (assert-validation-errors
            400
            [(msg/invalid-parent-collection-for-validation
               "Line 1 - cvc-elt.1: Cannot find the declaration of element 'Granule'.")]
            (d/item->concept (dg/granule-with-umm-spec-collection collection (:concept-id collection)))
            (assoc coll-concept :metadata invalid-granule-xml)))

        (testing "invalid granule xml"
          (assert-validation-errors
            400
            expected-errors
            (-> (dg/granule-with-umm-spec-collection collection (:concept-id collection))
                d/item->concept
                (assoc :metadata invalid-granule-xml))
            coll-concept))

        (testing "invalid multipart params"
          (let [response (ingest/multipart-param-request
                           (url/validate-url "PROV1" :granule "native-id")<commit_msg>CMR-5750 More test fixes.<commit_after>(ns cmr.system-int-test.ingest.granule-validation-test
  "CMR Ingest granule validation integration tests"
  (:require
    [clj-time.core :as t]
    [clj-time.format :as tf ]
    [clojure.java.io :as io]
    [clojure.java.shell :as shell]
    [clojure.test :refer :all]
    [cmr.common.time-keeper :as tk]
    [cmr.common.util :refer [are2]]
    [cmr.ingest.services.messages :as msg]
    [cmr.spatial.line-string :as l]
    [cmr.spatial.mbr :as m]
    [cmr.spatial.point :as p]
    [cmr.spatial.polygon :as poly]
    [cmr.system-int-test.data2.core :as d]
    [cmr.system-int-test.data2.granule :as dg]
    [cmr.system-int-test.data2.umm-spec-collection :as data-umm-c]
    [cmr.system-int-test.data2.umm-spec-common :as data-umm-cmn]
    [cmr.system-int-test.utils.dev-system-util :as dev-sys-util]
    [cmr.system-int-test.utils.ingest-util :as ingest]
    [cmr.system-int-test.utils.url-helper :as url]
    [cmr.umm.umm-granule :as umm-g]
    [cmr.umm.umm-spatial :as umm-s]))

(use-fixtures :each (ingest/reset-fixture {"provguid1" "PROV1" "provguid2" "PROV2"}))

(comment
  (do
    (dev-sys-util/reset)
    (ingest/create-provider {:provider-guid "provguid1" :provider-id "PROV1"}))

  (d/ingest-umm-spec-collection "PROV1" (data-umm-c/collection
                      {:AdditionalAttributes
                       [(data-umm-cmn/additional-attribute {:Name "bool" :DataType :boolean :value true})
                        (data-umm-cmn/additional-attribute {:Name "bool" :DataType :boolean :Value true})]})))



(defn assert-validation-errors
  "Asserts that when the granule and optional collection concept are validated the expected errors
  are returned. The collection concept can be passed as a third argument and it will be sent along
  with the granule instead of using a previously ingested collection."
  [expected-status-code expected-errors & gran-and-optional-coll-concept]
  (let [response (apply ingest/validate-granule gran-and-optional-coll-concept)]
    (is (= {:status expected-status-code :errors expected-errors}
           (select-keys response [:status :errors])))))

(defn assert-validation-success
  "Asserts that when the granule and optional collection concept are valid. The collection concept
  can be passed as a third argument and it will be sent along with the granule instead of using a
  previously ingested collection."
  [& gran-and-optional-coll-concept]
  (let [response (apply ingest/validate-granule gran-and-optional-coll-concept)]
    (is (= {:status 200} (select-keys response [:status :errors])))))

(deftest validation-endpoint-test
  (let [invalid-granule-xml "<Granule>invalid xml</Granule>"
        expected-errors ["Exception while parsing invalid XML: Line 1 - cvc-complex-type.2.3: Element 'Granule' cannot have character [children], because the type's content type is element-only."
                         "Exception while parsing invalid XML: Line 1 - cvc-complex-type.2.4.b: The content of element 'Granule' is not complete. One of '{GranuleUR}' is expected."]]

    (testing "granule with end date slightly in the future and collection with
             EndsAtPresentFlag=true and end date in the far future (see CMR-1351)"
             (let [collection (data-umm-c/collection {:EntryTitle "correct"
                                              :ShortName "S1"
                                              :Version "V1"
                                              :TemporalExtents [(data-umm-cmn/temporal-extent
                                                          {:beginning-date-time "2009-03-08T00:00:00Z"
                                                           :ending-date-time "2121-11-04T00:00:00Z"
                                                           :ends-at-present? true})]})
                   coll-concept (d/umm-c-collection->concept collection)
                   future-date-time (t/plus (tk/now) (t/hours 1))
                   time-formatter (tf/formatters :date-time-no-ms)
                   future-date-time-str (tf/unparse time-formatter future-date-time)
                   granule (assoc (dg/granule-with-umm-spec-collection collection (:concept-id collection))
                                  :temporal (dg/temporal {:beginning-date-time "2010-11-04T00:00:00Z"
                                                          :ending-date-time future-date-time-str}))]
               (assert-validation-success
                 (d/item->concept granule)
                 coll-concept)))

    (testing "with collection as additional parameter"
      (let [collection (data-umm-c/collection {})
            coll-concept (d/umm-c-collection->concept collection :echo10)]
        (testing "success"
          (let [concept (d/item->concept (dg/granule-with-umm-spec-collection collection (:concept-id collection)))]
            (assert-validation-success concept coll-concept)))
        (testing "collection in different format than granule"
          (let [concept (d/item->concept (dg/granule-with-umm-spec-collection collection (:concept-id collection)))]
            (assert-validation-success concept (d/umm-c-collection->concept collection :iso19115))
            (assert-validation-success concept (d/umm-c-collection->concept collection :dif))))

        (testing "invalid collection xml"
          (assert-validation-errors
            400
            [(msg/invalid-parent-collection-for-validation
               "Exception while parsing invalid XML: Line 1 - cvc-elt.1: Cannot find the declaration of element 'Granule'.")]
            (d/item->concept (dg/granule-with-umm-spec-collection collection (:concept-id collection)))
            (assoc coll-concept :metadata invalid-granule-xml)))

        (testing "invalid granule xml"
          (assert-validation-errors
            400
            expected-errors
            (-> (dg/granule-with-umm-spec-collection collection (:concept-id collection))
                d/item->concept
                (assoc :metadata invalid-granule-xml))
            coll-concept))

        (testing "invalid multipart params"
          (let [response (ingest/multipart-param-request
                           (url/validate-url "PROV1" :granule "native-id")
{'fraction_non_alphanumeric': 0.11938486442735735, 'fraction_numerical': 0.01424524484014569, 'mean_word_length': 2.221903520208605, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 12}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 30, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2017936', 'n_tokens_mistral': 3796, 'n_tokens_neox': 3505, 'n_words': 879}