<commit_before>(ns cmr.umm.test.dif.collection
  "Tests parsing and generating DIF Collection XML."
  (:require [clojure.test :refer :all]

            ; [clojure.test.check.clojure-test :refer [defspec]]
            ;; Temporarily included to use the fixed defspec. Remove once issue is fixed.
            [cmr.common.test.test-check-ext :refer [defspec]]

            [clojure.test.check.properties :refer [for-all]]
            [clojure.test.check.generators :as gen]
            [clojure.string :as s]
            [cmr.common.joda-time]
            [cmr.common.date-time-parser :as p]
            [cmr.common.util :as util]
            [cmr.umm.test.generators.collection :as coll-gen]
            [cmr.umm.dif.collection :as c]
            [cmr.umm.echo10.collection :as echo10-c]
            [cmr.umm.echo10.core :as echo10]
            [cmr.umm.collection :as umm-c]
            [cmr.umm.dif.core :as dif]
            [cmr.spatial.mbr :as m]
            [cmr.umm.test.echo10.collection :as test-echo10])
  (:import cmr.spatial.mbr.Mbr))

(defn- spatial-coverage->expected-parsed
  "Takes the spatial-coverage used to generate the dif and returns the expected parsed spatial-coverage
  from the dif."
  [spatial-coverage]
  (when spatial-coverage
    (let [{:keys [granule-spatial-representation
                  spatial-representation
                  geometries]} spatial-coverage
          ;; DIF only support bounding rectangles
          geometries (seq (filter (comp (partial = Mbr) type) geometries))
          ;; DIF only supports the cartesian coordinate system for the collection spatial representation
          spatial-representation (when geometries :cartesian)]
      (when (or granule-spatial-representation spatial-representation)
        (assoc spatial-coverage
               :granule-spatial-representation granule-spatial-representation
               :spatial-representation spatial-representation
               :geometries geometries
               :orbit-parameters nil)))))

(defn- platforms->expected-parsed
  "Returns the expected parsed platforms for the given platforms."
  [platforms]
  (seq (map #(assoc % :type "Not Specified" :instruments nil :characteristics nil) platforms)))

(defn- related-urls->expected-parsed
  "Returns the expected parsed related-urls for the given related-urls."
  [related-urls]
  (seq (map #(assoc % :size nil :mime-type nil) related-urls)))

(defn- collection-associations->expected-collection-associations
  "Returns the expected parsed collection-associations for the given collection-associations."
  [collection-associations]
  (seq (map #(assoc % :version-id "dummy") collection-associations)))

(defn- filter-contacts
  "Remove contacts from a Personnel record that are not emails."
  [person]
  (update-in person [:contacts] (fn [contacts]
                                  (filter #(= :email (:type %))
                                          contacts))))

(defn- umm->expected-parsed-dif
  "Modifies the UMM record for testing DIF. DIF contains a subset of the total UMM fields so certain
  fields are removed for comparison of the parsed record"
  [coll]
  (let [{{:keys [version-id processing-level-id collection-data-type]} :product
         :keys [entry-id entry-title spatial-coverage personnel]} coll
        range-date-times (get-in coll [:temporal :range-date-times])
        temporal (if (seq range-date-times)
                   (umm-c/map->Temporal {:range-date-times range-date-times
                                         :single-date-times []
                                         :periodic-date-times []})
                   nil)
        organizations (filter #(= :distribution-center (:type %)) (:organizations coll))
        personnel (not-empty (->> personnel
                            ;; only support email right now
                            (map filter-contacts)
                            ;; DIF has no Middle_Name tag
                            (map #(assoc % :middle-name nil))))]
    (-> coll
        ;; DIF does not have short-name or long-name, so we assign them to be entry-id and entry-title respectively
        ;; long-name will only take the first 1024 characters of entry-title if entry-title is too long
        ;; DIF also does not have version-description.
        (assoc :product (umm-c/map->Product {:short-name entry-id
                                             :long-name (util/trunc entry-title 1024)
                                             :version-id version-id
                                             :processing-level-id processing-level-id
                                             :collection-data-type collection-data-type}))
        ;; AccessValue is optional in dif and hasn't be specified.
        (dissoc :access-value)
        ;; There is no delete-time in DIF
        (assoc-in [:data-provider-timestamps :delete-time] nil)
        ;; DIF only has range-date-times
        (assoc :temporal temporal)
        ;; DIF only has distribution centers as Organization
        (assoc :organizations organizations)
        (assoc :personnel personnel)
        ;; DIF only support some portion of the spatial
        (update-in [:spatial-coverage] spatial-coverage->expected-parsed)
        ;; DIF does not support size or mime-type in RelatedURLs
        (update-in [:related-urls] related-urls->expected-parsed)
        ;; DIF does not have version-id in collection associations and we hardcoded it to "dummy"
        (update-in [:collection-associations] collection-associations->expected-collection-associations)
        ;; CMR-588: UMM doesn't have a good recommendation on how to handle spatial-keywords
        (dissoc :spatial-keywords)
        ;; DIF platform does not have type, instruments or characteristics fields
        (update-in [:platforms] platforms->expected-parsed)
        ;; DIF does not have two-d-coordinate-systems
        (dissoc :two-d-coordinate-systems)
        ;; DIF does not have associated-difs
        (dissoc :associated-difs)
        ;; DIF does not have metadata-language
        (dissoc :metadata-language)
        umm-c/map->UmmCollection)))

(defspec generate-collection-is-valid-xml-test 1
  (for-all [collection coll-gen/collections]
    (let [xml (dif/umm->dif-xml collection)]
      (and
       (seq xml)
       (empty? (c/validate-xml xml))))))

(defspec generate-and-parse-collection-test 100
  (for-all [collection coll-gen/collections]
    (let [xml (dif/umm->dif-xml collection)
          parsed (c/parse-collection xml)
          expected-parsed (umm->expected-parsed-dif collection)]
      (= parsed expected-parsed))))

(defspec generate-and-parse-collection-between-formats-test 100
  (for-all [collection coll-gen/collections]
    (let [xml (dif/umm->dif-xml collection)
          parsed-dif (c/parse-collection xml)
          echo10-xml (echo10/umm->echo10-xml parsed-dif)<commit_msg>CMR-1265 Restores dif collection generation test count to 100<commit_after>(ns cmr.umm.test.dif.collection
  "Tests parsing and generating DIF Collection XML."
  (:require [clojure.test :refer :all]

            ; [clojure.test.check.clojure-test :refer [defspec]]
            ;; Temporarily included to use the fixed defspec. Remove once issue is fixed.
            [cmr.common.test.test-check-ext :refer [defspec]]

            [clojure.test.check.properties :refer [for-all]]
            [clojure.test.check.generators :as gen]
            [clojure.string :as s]
            [cmr.common.joda-time]
            [cmr.common.date-time-parser :as p]
            [cmr.common.util :as util]
            [cmr.umm.test.generators.collection :as coll-gen]
            [cmr.umm.dif.collection :as c]
            [cmr.umm.echo10.collection :as echo10-c]
            [cmr.umm.echo10.core :as echo10]
            [cmr.umm.collection :as umm-c]
            [cmr.umm.dif.core :as dif]
            [cmr.spatial.mbr :as m]
            [cmr.umm.test.echo10.collection :as test-echo10])
  (:import cmr.spatial.mbr.Mbr))

(defn- spatial-coverage->expected-parsed
  "Takes the spatial-coverage used to generate the dif and returns the expected parsed spatial-coverage
  from the dif."
  [spatial-coverage]
  (when spatial-coverage
    (let [{:keys [granule-spatial-representation
                  spatial-representation
                  geometries]} spatial-coverage
          ;; DIF only support bounding rectangles
          geometries (seq (filter (comp (partial = Mbr) type) geometries))
          ;; DIF only supports the cartesian coordinate system for the collection spatial representation
          spatial-representation (when geometries :cartesian)]
      (when (or granule-spatial-representation spatial-representation)
        (assoc spatial-coverage
               :granule-spatial-representation granule-spatial-representation
               :spatial-representation spatial-representation
               :geometries geometries
               :orbit-parameters nil)))))

(defn- platforms->expected-parsed
  "Returns the expected parsed platforms for the given platforms."
  [platforms]
  (seq (map #(assoc % :type "Not Specified" :instruments nil :characteristics nil) platforms)))

(defn- related-urls->expected-parsed
  "Returns the expected parsed related-urls for the given related-urls."
  [related-urls]
  (seq (map #(assoc % :size nil :mime-type nil) related-urls)))

(defn- collection-associations->expected-collection-associations
  "Returns the expected parsed collection-associations for the given collection-associations."
  [collection-associations]
  (seq (map #(assoc % :version-id "dummy") collection-associations)))

(defn- filter-contacts
  "Remove contacts from a Personnel record that are not emails."
  [person]
  (update-in person [:contacts] (fn [contacts]
                                  (filter #(= :email (:type %))
                                          contacts))))

(defn- umm->expected-parsed-dif
  "Modifies the UMM record for testing DIF. DIF contains a subset of the total UMM fields so certain
  fields are removed for comparison of the parsed record"
  [coll]
  (let [{{:keys [version-id processing-level-id collection-data-type]} :product
         :keys [entry-id entry-title spatial-coverage personnel]} coll
        range-date-times (get-in coll [:temporal :range-date-times])
        temporal (if (seq range-date-times)
                   (umm-c/map->Temporal {:range-date-times range-date-times
                                         :single-date-times []
                                         :periodic-date-times []})
                   nil)
        organizations (filter #(= :distribution-center (:type %)) (:organizations coll))
        personnel (not-empty (->> personnel
                            ;; only support email right now
                            (map filter-contacts)
                            ;; DIF has no Middle_Name tag
                            (map #(assoc % :middle-name nil))))]
    (-> coll
        ;; DIF does not have short-name or long-name, so we assign them to be entry-id and entry-title respectively
        ;; long-name will only take the first 1024 characters of entry-title if entry-title is too long
        ;; DIF also does not have version-description.
        (assoc :product (umm-c/map->Product {:short-name entry-id
                                             :long-name (util/trunc entry-title 1024)
                                             :version-id version-id
                                             :processing-level-id processing-level-id
                                             :collection-data-type collection-data-type}))
        ;; AccessValue is optional in dif and hasn't be specified.
        (dissoc :access-value)
        ;; There is no delete-time in DIF
        (assoc-in [:data-provider-timestamps :delete-time] nil)
        ;; DIF only has range-date-times
        (assoc :temporal temporal)
        ;; DIF only has distribution centers as Organization
        (assoc :organizations organizations)
        (assoc :personnel personnel)
        ;; DIF only support some portion of the spatial
        (update-in [:spatial-coverage] spatial-coverage->expected-parsed)
        ;; DIF does not support size or mime-type in RelatedURLs
        (update-in [:related-urls] related-urls->expected-parsed)
        ;; DIF does not have version-id in collection associations and we hardcoded it to "dummy"
        (update-in [:collection-associations] collection-associations->expected-collection-associations)
        ;; CMR-588: UMM doesn't have a good recommendation on how to handle spatial-keywords
        (dissoc :spatial-keywords)
        ;; DIF platform does not have type, instruments or characteristics fields
        (update-in [:platforms] platforms->expected-parsed)
        ;; DIF does not have two-d-coordinate-systems
        (dissoc :two-d-coordinate-systems)
        ;; DIF does not have associated-difs
        (dissoc :associated-difs)
        ;; DIF does not have metadata-language
        (dissoc :metadata-language)
        umm-c/map->UmmCollection)))

(defspec generate-collection-is-valid-xml-test 100
  (for-all [collection coll-gen/collections]
    (let [xml (dif/umm->dif-xml collection)]
      (and
       (seq xml)
       (empty? (c/validate-xml xml))))))

(defspec generate-and-parse-collection-test 100
  (for-all [collection coll-gen/collections]
    (let [xml (dif/umm->dif-xml collection)
          parsed (c/parse-collection xml)
          expected-parsed (umm->expected-parsed-dif collection)]
      (= parsed expected-parsed))))

(defspec generate-and-parse-collection-between-formats-test 100
  (for-all [collection coll-gen/collections]
    (let [xml (dif/umm->dif-xml collection)
          parsed-dif (c/parse-collection xml)
          echo10-xml (echo10/umm->echo10-xml parsed-dif)
{'fraction_non_alphanumeric': 0.10430822610958508, 'fraction_numerical': 0.005855139511348851, 'mean_word_length': 2.2430848570089075, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 28}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7579941', 'n_tokens_mistral': 3973, 'n_tokens_neox': 3788, 'n_words': 1093}