<commit_before>(ns cmr.indexer.services.concepts.granule
  "Contains functions to parse and convert granule concept"
  (:require [clojure.string :as s]
            [clj-time.format :as f]
            [cmr.indexer.services.index-service :as idx]
            [cmr.umm.echo10.granule :as granule]
            [cmr.umm.echo10.related-url :as ru]
            [cmr.transmit.metadata-db :as mdb]
            [cmr.common.services.errors :as errors]
            [cmr.indexer.services.concepts.temporal :as temporal]
            [cmr.indexer.services.concepts.attribute :as attrib]
            [cmr.indexer.services.concepts.orbit-calculated-spatial-domain :as ocsd]
            [cmr.indexer.services.concepts.spatial :as spatial]
            [cmr.common.cache :as cache]))

(defmethod idx/parse-concept :granule
  [concept]
  (granule/parse-granule (:metadata concept)))

(defn- fetch-parent-collection
  "Retrieve the parent collection umm from the db"
  [context parent-collection-id]
  (let [concept (mdb/get-latest-concept context parent-collection-id)]
    (assoc (idx/parse-concept concept) :concept-id parent-collection-id)))

(defn- get-parent-collection
  [context parent-collection-id]
  (if-let [cache (get-in context [:system :parent-collection-cache])]
    (cache/cache-lookup cache parent-collection-id
                        (partial fetch-parent-collection context parent-collection-id))
    (fetch-parent-collection context parent-collection-id)))

(defn spatial->elastic
  [parent-collection granule]
  (when (get-in granule [:spatial-coverage :geometries])
    (let [gsr (get-in parent-collection [:spatial-coverage :granule-spatial-representation])]
      (if (= gsr :geodetic)
        (spatial/spatial->elastic-docs gsr granule)
        (errors/internal-error!
          (format "Only geodetic is supported for granule spatial representation not [%s]"
                  gsr))))))

(defmethod idx/concept->elastic-doc :granule
  [context concept umm-granule]
  (let [{:keys [concept-id extra-fields provider-id revision-date]} concept
        {:keys [parent-collection-id]} extra-fields
        parent-collection (get-parent-collection context parent-collection-id)
        {:keys [granule-ur data-granule temporal platform-refs project-refs related-urls cloud-cover]} umm-granule
        {:keys [size producer-gran-id day-night]} data-granule
        platform-short-names (map :short-name platform-refs)
        instrument-refs (mapcat :instrument-refs platform-refs)
        instrument-short-names (remove nil? (map :short-name instrument-refs))
        sensor-refs (mapcat :sensor-refs instrument-refs)
        sensor-short-names (remove nil? (map :short-name sensor-refs))
        start-date (temporal/start-date :granule temporal)
        end-date (temporal/end-date :granule temporal)
        downloadable (not (empty? (filter ru/downloadable-url? related-urls)))]
    (merge {:concept-id concept-id
            :collection-concept-id parent-collection-id

            :entry-title.lowercase (s/lower-case (:entry-title parent-collection))
            :short-name.lowercase (s/lower-case (get-in parent-collection [:product :short-name]))
            :version-id.lowercase (s/lower-case (get-in parent-collection [:product :version-id]))

            :provider-id provider-id
            :provider-id.lowercase (s/lower-case provider-id)
            :granule-ur granule-ur
            :granule-ur.lowercase (s/lower-case granule-ur)<commit_msg>Changed granule spatial indexing so coordinate system types other than geodetic do not throw errors. We just log an error and ignore spatial. We need this so that we can index and test in workload.<commit_after>(ns cmr.indexer.services.concepts.granule
  "Contains functions to parse and convert granule concept"
  (:require [clojure.string :as s]
            [clj-time.format :as f]
            [cmr.indexer.services.index-service :as idx]
            [cmr.umm.echo10.granule :as granule]
            [cmr.umm.echo10.related-url :as ru]
            [cmr.transmit.metadata-db :as mdb]
            [cmr.common.log :refer (debug info warn error)]
            [cmr.common.services.errors :as errors]
            [cmr.indexer.services.concepts.temporal :as temporal]
            [cmr.indexer.services.concepts.attribute :as attrib]
            [cmr.indexer.services.concepts.orbit-calculated-spatial-domain :as ocsd]
            [cmr.indexer.services.concepts.spatial :as spatial]
            [cmr.common.cache :as cache]))

(defmethod idx/parse-concept :granule
  [concept]
  (granule/parse-granule (:metadata concept)))

(defn- fetch-parent-collection
  "Retrieve the parent collection umm from the db"
  [context parent-collection-id]
  (let [parent-collection-cache (get-in context [:system :parent-collection-cache])
        concept (mdb/get-latest-concept context parent-collection-id)]
    (assoc (idx/parse-concept concept) :concept-id parent-collection-id)))

(defn- get-parent-collection
  [context parent-collection-id]
  (if-let [cache (get-in context [:system :parent-collection-cache])]
    (cache/cache-lookup cache parent-collection-id
                        (partial fetch-parent-collection context parent-collection-id))
    (fetch-parent-collection context parent-collection-id)))

(defn spatial->elastic
  [parent-collection granule]
  (when (get-in granule [:spatial-coverage :geometries])
    (let [gsr (get-in parent-collection [:spatial-coverage :granule-spatial-representation])]
      (if (= gsr :geodetic)
        (spatial/spatial->elastic-docs gsr granule)
        (error
          (format "Only geodetic is supported for granule spatial representation currently. GSR: [%s]"
                  gsr))))))

(defmethod idx/concept->elastic-doc :granule
  [context concept umm-granule]
  (let [{:keys [concept-id extra-fields provider-id revision-date]} concept
        {:keys [parent-collection-id]} extra-fields
        parent-collection (get-parent-collection context parent-collection-id)
        {:keys [granule-ur data-granule temporal platform-refs project-refs related-urls cloud-cover]} umm-granule
        {:keys [size producer-gran-id day-night]} data-granule
        platform-short-names (map :short-name platform-refs)
        instrument-refs (mapcat :instrument-refs platform-refs)
        instrument-short-names (remove nil? (map :short-name instrument-refs))
        sensor-refs (mapcat :sensor-refs instrument-refs)
        sensor-short-names (remove nil? (map :short-name sensor-refs))
        start-date (temporal/start-date :granule temporal)
        end-date (temporal/end-date :granule temporal)
        downloadable (not (empty? (filter ru/downloadable-url? related-urls)))]
    (merge {:concept-id concept-id
            :collection-concept-id parent-collection-id

            :entry-title.lowercase (s/lower-case (:entry-title parent-collection))
            :short-name.lowercase (s/lower-case (get-in parent-collection [:product :short-name]))
            :version-id.lowercase (s/lower-case (get-in parent-collection [:product :version-id]))

            :provider-id provider-id
            :provider-id.lowercase (s/lower-case provider-id)
            :granule-ur granule-ur
            :granule-ur.lowercase (s/lower-case granule-ur)
{'fraction_non_alphanumeric': 0.12115358079205188, 'fraction_numerical': 0.0011039050641644817, 'mean_word_length': 2.960655737704918, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4246745', 'n_tokens_mistral': 2295, 'n_tokens_neox': 2153, 'n_words': 511}