<commit_before>(ns cmr.indexer.data.concepts.granule
  "Contains functions to parse and convert granule concept"
  (:require [clojure.string :as s]
            [clj-time.format :as f]
            [cheshire.core :as json]
            [camel-snake-kebab :as csk]
            [cmr.indexer.data.elasticsearch :as es]
            [cmr.umm.core :as umm]
            [cmr.umm.related-url-helper :as ru]
            [cmr.transmit.metadata-db :as mdb]
            [cmr.common.log :refer (debug info warn error)]
            [cmr.common.mime-types :as mt]
            [cmr.common.services.errors :as errors]
            [cmr.indexer.data.concepts.temporal :as temporal]
            [cmr.indexer.data.concepts.attribute :as attrib]
            [cmr.indexer.data.concepts.orbit-calculated-spatial-domain :as ocsd]
            [cmr.indexer.data.concepts.spatial :as spatial]
            [cmr.common.cache :as cache])
  (:import cmr.spatial.mbr.Mbr))

(defn unrecognized-gsr-msg
  "The granule spatial representaiton (gsr) is not of a known type."
  [gsr]
  (str "Unrecognized granule spatial represention [" gsr "]"))

(defn- fetch-parent-collection
  "Retrieve the parent collection umm from the db"
  [context parent-collection-id]
  (let [parent-collection-cache (get-in context [:system :parent-collection-cache])
        concept (mdb/get-latest-concept context parent-collection-id)]
    (assoc (umm/parse-concept concept) :concept-id parent-collection-id)))

(defn- get-parent-collection
  [context parent-collection-id]
  (if-let [cache (get-in context [:system :parent-collection-cache])]
    (cache/cache-lookup cache parent-collection-id
                        (partial fetch-parent-collection context parent-collection-id))
    (fetch-parent-collection context parent-collection-id)))

(defn orbit->circular-latitude-range
  "Compute a circular latitude range from the start and end latitudes of an orbit."
  [orbit]
  (let [{:keys [^double start-lat ^double end-lat start-direction end-direction]} orbit
        start-lat (if (= :desc start-direction)
                    (- 180.0 start-lat)
                    start-lat)
        end-lat (if (= :desc end-direction)
                  (- 180.0 end-lat)
                  end-lat)
        min (mod start-lat 360.0)
        max (if (= 360.0 (- end-lat start-lat))
              (+ min 360.0)
              (+ min (mod (- end-lat start-lat) 360.0)))]
    [min max]))

(defn spatial->elastic
  [parent-collection granule]
  (when-let [gsr (get-in parent-collection [:spatial-coverage :granule-spatial-representation])]
    (cond
      (or (= gsr :geodetic) (= gsr :cartesian))
      (let [geometries (seq (get-in granule [:spatial-coverage :geometries]))]
        (spatial/spatial->elastic-docs gsr granule))

      (= gsr :no-spatial)
      nil

      (= gsr :orbit)
      (let [orbit (get-in granule [:spatial-coverage :orbit])
            orbit-asc-crossing-lon (:ascending-crossing orbit)
            [^double orbit-start-clat ^double orbit-end-clat] (orbit->circular-latitude-range orbit)]
        {:orbit-asc-crossing-lon orbit-asc-crossing-lon
         :orbit-start-clat orbit-start-clat
         :orbit-end-clat (if (= orbit-end-clat orbit-start-clat)
                           (+ orbit-end-clat 360.0)
                           orbit-end-clat)})
      :else
      (errors/throw-service-error :invalid-data (unrecognized-gsr-msg gsr)))))

(def ocsd-fields
  "The fields for orbit calculated spatil domains, in the order that they are stored in the jason
  string in the index."
  [:equator-crossing-date-time
   :equator-crossing-longitude
   :orbital-model-name
   :orbit-number
   :start-orbit-number
   :stop-orbit-number])

(defn- ocsd-map->vector
  "Turn a map of orbit crossing spatial domain data into a vector"
  [ocsd]
  (map ocsd ocsd-fields))

(defn- granule->ocsd-json
  "Create a json string from the orbitial calculated spatial domains."
  [umm-granule]
  (map #(json/generate-string
                          (ocsd-map->vector %))
                       (ocsd/ocsds->elastic-docs umm-granule)))

(defmethod es/concept->elastic-doc :granule
  [context concept umm-granule]
  (let [{:keys [concept-id extra-fields provider-id revision-date format]} concept
        {:keys [parent-collection-id]} extra-fields
        parent-collection (get-parent-collection context parent-collection-id)
        {:keys [granule-ur data-granule temporal platform-refs project-refs related-urls cloud-cover
                access-value two-d-coordinate-system]} umm-granule
        {:keys [size producer-gran-id day-night]} data-granule
        {:keys [start-coordinate-1 end-coordinate-1 start-coordinate-2 end-coordinate-2]
         two-d-coord-name :name} two-d-coordinate-system
        platform-short-names (map :short-name platform-refs)
        instrument-refs (mapcat :instrument-refs platform-refs)
        instrument-short-names (remove nil? (map :short-name instrument-refs))
        sensor-refs (mapcat :sensor-refs instrument-refs)
        sensor-short-names (remove nil? (map :short-name sensor-refs))
        start-date (temporal/start-date :granule temporal)
        end-date (temporal/end-date :granule temporal)
        atom-links (map json/generate-string (ru/atom-links related-urls))
        ocsd-json (granule->ocsd-json umm-granule)
        ;; not empty is used below to get a real true false value
        downloadable (not (empty? (ru/downloadable-urls related-urls)))
        browsable (not (empty? (ru/browse-urls related-urls)))
        update-time (get-in umm-granule [:data-provider-timestamps :update-time])
        update-time (f/unparse (f/formatters :date-time) update-time)
        {:keys [short-name version-id]} (:product parent-collection)
        granule-spatial-representation (get-in parent-collection [:spatial-coverage :granule-spatial-representation])]
    (merge {:concept-id concept-id
            :collection-concept-id parent-collection-id

            :entry-title (:entry-title parent-collection)
            :original-format (s/upper-case (name (mt/mime-type->format format)))
            :update-time update-time
            :coordinate-system (when granule-spatial-representation (csk/->SNAKE_CASE_STRING granule-spatial-representation))

            :entry-title.lowercase (s/lower-case (:entry-title parent-collection))
            :short-name.lowercase (when short-name (s/lower-case short-name))
            :version-id.lowercase (when version-id (s/lower-case version-id))

            :provider-id provider-id
            :provider-id.lowercase (s/lower-case provider-id)
            :granule-ur granule-ur
            :granule-ur.lowercase (s/lower-case granule-ur)
            :producer-gran-id producer-gran-id
            :producer-gran-id.lowercase (when producer-gran-id (s/lower-case producer-gran-id))
            :day-night day-night
            :day-night.lowercase (when day-night (s/lower-case day-night))
            :access-value access-value

            ;; Provides sorting on a combination of producer granule id and granule ur
            :readable-granule-name-sort (s/lower-case (or producer-gran-id granule-ur))

            :platform-sn platform-short-names
            :platform-sn.lowercase  (map s/lower-case platform-short-names)
            :instrument-sn instrument-short-names
            :instrument-sn.lowercase  (map s/lower-case instrument-short-names)
            :sensor-sn sensor-short-names
            :sensor-sn.lowercase  (map s/lower-case sensor-short-names)
            :project-refs project-refs
            :project-refs.lowercase (map s/lower-case project-refs)
            :size size
            :cloud-cover cloud-cover
            :orbit-calculated-spatial-domains (ocsd/ocsds->elastic-docs umm-granule)
            :attributes (attrib/psa-refs->elastic-docs parent-collection umm-granule)
            :revision-date revision-date
            :downloadable downloadable
            :browsable browsable
            :start-date (when start-date (f/unparse (f/formatters :date-time) start-date))
            :end-date (when end-date (f/unparse (f/formatters :date-time) end-date))
            :two-d-coord-name.lowercase (when two-d-coord-name (s/lower-case two-d-coord-name))
            :start-coordinate-1 start-coordinate-1<commit_msg>Fixed an error caused in resolve merge conflict.<commit_after>(ns cmr.indexer.data.concepts.granule
  "Contains functions to parse and convert granule concept"
  (:require [clojure.string :as s]
            [clj-time.format :as f]
            [cheshire.core :as json]
            [camel-snake-kebab :as csk]
            [cmr.indexer.data.elasticsearch :as es]
            [cmr.umm.core :as umm]
            [cmr.umm.related-url-helper :as ru]
            [cmr.transmit.metadata-db :as mdb]
            [cmr.common.log :refer (debug info warn error)]
            [cmr.common.mime-types :as mt]
            [cmr.common.services.errors :as errors]
            [cmr.indexer.data.concepts.temporal :as temporal]
            [cmr.indexer.data.concepts.attribute :as attrib]
            [cmr.indexer.data.concepts.orbit-calculated-spatial-domain :as ocsd]
            [cmr.indexer.data.concepts.spatial :as spatial]
            [cmr.common.cache :as cache])
  (:import cmr.spatial.mbr.Mbr))

(defn unrecognized-gsr-msg
  "The granule spatial representaiton (gsr) is not of a known type."
  [gsr]
  (str "Unrecognized granule spatial represention [" gsr "]"))

(defn- fetch-parent-collection
  "Retrieve the parent collection umm from the db"
  [context parent-collection-id]
  (let [parent-collection-cache (get-in context [:system :parent-collection-cache])
        concept (mdb/get-latest-concept context parent-collection-id)]
    (assoc (umm/parse-concept concept) :concept-id parent-collection-id)))

(defn- get-parent-collection
  [context parent-collection-id]
  (if-let [cache (get-in context [:system :parent-collection-cache])]
    (cache/cache-lookup cache parent-collection-id
                        (partial fetch-parent-collection context parent-collection-id))
    (fetch-parent-collection context parent-collection-id)))

(defn orbit->circular-latitude-range
  "Compute a circular latitude range from the start and end latitudes of an orbit."
  [orbit]
  (let [{:keys [^double start-lat ^double end-lat start-direction end-direction]} orbit
        start-lat (if (= :desc start-direction)
                    (- 180.0 start-lat)
                    start-lat)
        end-lat (if (= :desc end-direction)
                  (- 180.0 end-lat)
                  end-lat)
        min (mod start-lat 360.0)
        max (if (= 360.0 (- end-lat start-lat))
              (+ min 360.0)
              (+ min (mod (- end-lat start-lat) 360.0)))]
    [min max]))

(defn spatial->elastic
  [parent-collection granule]
  (when-let [gsr (get-in parent-collection [:spatial-coverage :granule-spatial-representation])]
    (cond
      (or (= gsr :geodetic) (= gsr :cartesian))
      (let [geometries (seq (get-in granule [:spatial-coverage :geometries]))]
        (spatial/spatial->elastic-docs gsr granule))

      (= gsr :no-spatial)
      nil

      (= gsr :orbit)
      (let [orbit (get-in granule [:spatial-coverage :orbit])
            orbit-asc-crossing-lon (:ascending-crossing orbit)
            [^double orbit-start-clat ^double orbit-end-clat] (orbit->circular-latitude-range orbit)]
        {:orbit-asc-crossing-lon orbit-asc-crossing-lon
         :orbit-start-clat orbit-start-clat
         :orbit-end-clat (if (= orbit-end-clat orbit-start-clat)
                           (+ orbit-end-clat 360.0)
                           orbit-end-clat)})
      :else
      (errors/throw-service-error :invalid-data (unrecognized-gsr-msg gsr)))))

(def ocsd-fields
  "The fields for orbit calculated spatil domains, in the order that they are stored in the jason
  string in the index."
  [:equator-crossing-date-time
   :equator-crossing-longitude
   :orbital-model-name
   :orbit-number
   :start-orbit-number
   :stop-orbit-number])

(defn- ocsd-map->vector
  "Turn a map of orbit crossing spatial domain data into a vector"
  [ocsd]
  (map ocsd ocsd-fields))

(defn- granule->ocsd-json
  "Create a json string from the orbitial calculated spatial domains."
  [umm-granule]
  (map #(json/generate-string
                          (ocsd-map->vector %))
                       (ocsd/ocsds->elastic-docs umm-granule)))

(defmethod es/concept->elastic-doc :granule
  [context concept umm-granule]
  (let [{:keys [concept-id extra-fields provider-id revision-date format]} concept
        {:keys [parent-collection-id]} extra-fields
        parent-collection (get-parent-collection context parent-collection-id)
        {:keys [granule-ur data-granule temporal platform-refs project-refs related-urls cloud-cover
                access-value two-d-coordinate-system]} umm-granule
        {:keys [size producer-gran-id day-night]} data-granule
        {:keys [start-coordinate-1 end-coordinate-1 start-coordinate-2 end-coordinate-2]
         two-d-coord-name :name} two-d-coordinate-system
        platform-short-names (map :short-name platform-refs)
        instrument-refs (mapcat :instrument-refs platform-refs)
        instrument-short-names (remove nil? (map :short-name instrument-refs))
        sensor-refs (mapcat :sensor-refs instrument-refs)
        sensor-short-names (remove nil? (map :short-name sensor-refs))
        start-date (temporal/start-date :granule temporal)
        end-date (temporal/end-date :granule temporal)
        atom-links (map json/generate-string (ru/atom-links related-urls))
        ocsd-json (granule->ocsd-json umm-granule)
        ;; not empty is used below to get a real true false value
        downloadable (not (empty? (ru/downloadable-urls related-urls)))
        browsable (not (empty? (ru/browse-urls related-urls)))
        update-time (get-in umm-granule [:data-provider-timestamps :update-time])
        update-time (f/unparse (f/formatters :date-time) update-time)
        {:keys [short-name version-id]} (:product parent-collection)
        granule-spatial-representation (get-in parent-collection [:spatial-coverage :granule-spatial-representation])]
    (merge {:concept-id concept-id
            :collection-concept-id parent-collection-id

            :entry-title (:entry-title parent-collection)
            :original-format (s/upper-case (name (mt/mime-type->format format)))
            :update-time update-time
            :coordinate-system (when granule-spatial-representation (csk/->SNAKE_CASE_STRING granule-spatial-representation))

            :entry-title.lowercase (s/lower-case (:entry-title parent-collection))
            :short-name.lowercase (when short-name (s/lower-case short-name))
            :version-id.lowercase (when version-id (s/lower-case version-id))

            :provider-id provider-id
            :provider-id.lowercase (s/lower-case provider-id)
            :granule-ur granule-ur
            :granule-ur.lowercase (s/lower-case granule-ur)
            :producer-gran-id producer-gran-id
            :producer-gran-id.lowercase (when producer-gran-id (s/lower-case producer-gran-id))
            :day-night day-night
            :day-night.lowercase (when day-night (s/lower-case day-night))
            :access-value access-value

            ;; Provides sorting on a combination of producer granule id and granule ur
            :readable-granule-name-sort (s/lower-case (or producer-gran-id granule-ur))

            :platform-sn platform-short-names
            :platform-sn.lowercase  (map s/lower-case platform-short-names)
            :instrument-sn instrument-short-names
            :instrument-sn.lowercase  (map s/lower-case instrument-short-names)
            :sensor-sn sensor-short-names
            :sensor-sn.lowercase  (map s/lower-case sensor-short-names)
            :project-refs project-refs
            :project-refs.lowercase (map s/lower-case project-refs)
            :size size
            :cloud-cover cloud-cover
            :orbit-calculated-spatial-domains (ocsd/ocsds->elastic-docs umm-granule)
            :attributes (attrib/psa-refs->elastic-docs parent-collection umm-granule)
            :revision-date revision-date
            :downloadable downloadable
            :browsable browsable
            :start-date (when start-date (f/unparse (f/formatters :date-time) start-date))
            :end-date (when end-date (f/unparse (f/formatters :date-time) end-date))
            :two-d-coord-name two-d-coord-name
            :two-d-coord-name.lowercase (when two-d-coord-name (s/lower-case two-d-coord-name))
            :start-coordinate-1 start-coordinate-1
{'fraction_non_alphanumeric': 0.1230806142034549, 'fraction_numerical': 0.004078694817658349, 'mean_word_length': 2.445546600537301, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7475060', 'n_tokens_mistral': 5429, 'n_tokens_neox': 5080, 'n_words': 1172}