<commit_before>(ns cmr.system-int-test.data2.core
  "Contains helper functions for data generation and ingest for example based testing in system
  integration tests."
  (:require [clojure.test :refer [is]]
            [cmr.umm.core :as umm]
            [cmr.common.mime-types :as mime-types]
            [cmr.system-int-test.utils.ingest-util :as ingest]
            [cmr.system-int-test.utils.url-helper :as url]
            [clj-time.core :as t]
            [clj-time.format :as f]
            [cheshire.core :as json])
  (:import cmr.umm.collection.UmmCollection
           cmr.umm.granule.UmmGranule))

(defmulti item->native-id
  "Returns the native id of an item"
  (fn [item]
    (type item)))

(defmethod item->native-id UmmCollection
  [item]
  (:entry-title item))

(defmethod item->native-id UmmGranule
  [item]
  (:granule-ur item))

(defmulti item->concept-type
  "Returns the path to ingest the item"
  (fn [item]
    (type item)))

(defmethod item->concept-type UmmCollection
  [item]
  :collection)

(defmethod item->concept-type UmmGranule
  [item]
  :granule)

(defn item->concept
  "Converts an UMM item to a concept map. Expects provider-id to be in the item"
  [item format-key]
  (let [format (mime-types/format->mime-type format-key)]
    (merge {:concept-type (item->concept-type item)
            :provider-id (:provider-id item)
            :native-id (item->native-id item)
            :metadata (umm/umm->xml item format-key)
            :format format}
           (when (:concept-id item)
             {:concept-id (:concept-id item)})
           (when (:revision-id item)
             {:revision-id (:revision-id item)}))))

(defn ingest
  "Ingests the catalog item. Returns it with concept-id, revision-id, and provider-id set on it."
  ([provider-id item]
   (ingest provider-id item :echo10))
  ([provider-id item format-key]
   (let [response (ingest/ingest-concept
                    (item->concept (assoc item :provider-id provider-id) format-key))]
     (is (= 200 (:status response))
         (pr-str response))
     (assoc item
            :provider-id provider-id
            :concept-id (:concept-id response)
            :revision-id (:revision-id response)))))

(defn item->ref
  "Converts an item into the expected reference"
  [item]
  (let [{:keys [concept-id revision-id]} item]
    {:name (item->native-id item)
     :id concept-id
     :location (str (url/location-root) (:concept-id item))
     :revision-id revision-id}))

(defn item->metadata-result
  "Converts an item into the expected metadata result"
  [format-key item]
  (let [{:keys [concept-id revision-id collection-concept-id]} item]
    {:concept-id concept-id
     :revision-id revision-id
     :format format-key
     :collection-concept-id collection-concept-id
     :metadata (umm/umm->xml item format-key)}))

(defn metadata-results-match?
  "Returns true if the metadata results match the expected items"
  [format-key items search-result]
  (= (set (map (partial item->metadata-result format-key) items))
     (set search-result)))

(defn assert-metadata-results-match
  "Returns true if the metadata results match the expected items"
  [format-key items search-result]
  (is (= (set (map (partial item->metadata-result format-key) items))
         (set search-result))))

(defn refs-match?
  "Returns true if the references match the expected items"
  [items search-result]
  (let [result (= (set (map item->ref items))
                  (set (map #(dissoc % :score) (:refs search-result))))]
    (when (:status search-result)
      (println (pr-str search-result)))
    result))

(defn refs-match-order?
  "Returns true if the references match the expected items in the order specified"
  [items search-result]
  (= (map item->ref items)
     (map #(dissoc % :score) (:refs search-result))))

(defmacro record-fields
  "Returns the set of fields in a record type as keywords. The record type passed in must be a java
  class. Uses the getBasis function on record classes which returns a list of symbols of the fields of<commit_msg>CMR-657: Added comment to explain dissoc of score field.<commit_after>(ns cmr.system-int-test.data2.core
  "Contains helper functions for data generation and ingest for example based testing in system
  integration tests."
  (:require [clojure.test :refer [is]]
            [cmr.umm.core :as umm]
            [cmr.common.mime-types :as mime-types]
            [cmr.system-int-test.utils.ingest-util :as ingest]
            [cmr.system-int-test.utils.url-helper :as url]
            [clj-time.core :as t]
            [clj-time.format :as f]
            [cheshire.core :as json])
  (:import cmr.umm.collection.UmmCollection
           cmr.umm.granule.UmmGranule))

(defmulti item->native-id
  "Returns the native id of an item"
  (fn [item]
    (type item)))

(defmethod item->native-id UmmCollection
  [item]
  (:entry-title item))

(defmethod item->native-id UmmGranule
  [item]
  (:granule-ur item))

(defmulti item->concept-type
  "Returns the path to ingest the item"
  (fn [item]
    (type item)))

(defmethod item->concept-type UmmCollection
  [item]
  :collection)

(defmethod item->concept-type UmmGranule
  [item]
  :granule)

(defn item->concept
  "Converts an UMM item to a concept map. Expects provider-id to be in the item"
  [item format-key]
  (let [format (mime-types/format->mime-type format-key)]
    (merge {:concept-type (item->concept-type item)
            :provider-id (:provider-id item)
            :native-id (item->native-id item)
            :metadata (umm/umm->xml item format-key)
            :format format}
           (when (:concept-id item)
             {:concept-id (:concept-id item)})
           (when (:revision-id item)
             {:revision-id (:revision-id item)}))))

(defn ingest
  "Ingests the catalog item. Returns it with concept-id, revision-id, and provider-id set on it."
  ([provider-id item]
   (ingest provider-id item :echo10))
  ([provider-id item format-key]
   (let [response (ingest/ingest-concept
                    (item->concept (assoc item :provider-id provider-id) format-key))]
     (is (= 200 (:status response))
         (pr-str response))
     (assoc item
            :provider-id provider-id
            :concept-id (:concept-id response)
            :revision-id (:revision-id response)))))

(defn item->ref
  "Converts an item into the expected reference"
  [item]
  (let [{:keys [concept-id revision-id]} item]
    {:name (item->native-id item)
     :id concept-id
     :location (str (url/location-root) (:concept-id item))
     :revision-id revision-id}))

(defn item->metadata-result
  "Converts an item into the expected metadata result"
  [format-key item]
  (let [{:keys [concept-id revision-id collection-concept-id]} item]
    {:concept-id concept-id
     :revision-id revision-id
     :format format-key
     :collection-concept-id collection-concept-id
     :metadata (umm/umm->xml item format-key)}))

(defn metadata-results-match?
  "Returns true if the metadata results match the expected items"
  [format-key items search-result]
  (= (set (map (partial item->metadata-result format-key) items))
     (set search-result)))

(defn assert-metadata-results-match
  "Returns true if the metadata results match the expected items"
  [format-key items search-result]
  (is (= (set (map (partial item->metadata-result format-key) items))
         (set search-result))))

(defn refs-match?
  "Returns true if the references match the expected items"
  [items search-result]
  (let [result (= (set (map item->ref items))
                  ;; need to remove score because it won't be available in collections
                  ;; to which we are comparing
                  (set (map #(dissoc % :score) (:refs search-result))))]
    (when (:status search-result)
      (println (pr-str search-result)))
    result))

(defn refs-match-order?
  "Returns true if the references match the expected items in the order specified"
  [items search-result]
  (= (map item->ref items)
     ;; need to remove score because it won't be available in collections
     ;; to which we are comparing
     (map #(dissoc % :score) (:refs search-result))))

(defmacro record-fields
  "Returns the set of fields in a record type as keywords. The record type passed in must be a java
  class. Uses the getBasis function on record classes which returns a list of symbols of the fields of
{'fraction_non_alphanumeric': 0.12495533055390114, 'fraction_numerical': 0.001786777843954735, 'mean_word_length': 2.867342238599724, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4109598', 'n_tokens_mistral': 2719, 'n_tokens_neox': 2589, 'n_words': 831}