<commit_before>(ns cmr.indexer.services.concepts.granule
  "Contains functions to parse and convert granule concept"
  (:require [clojure.string :as s]
            [clj-time.format :as f]
            [cmr.indexer.services.index-service :as idx]
            [cmr.umm.echo10.granule :as granule]
            [cmr.umm.echo10.related-url :as ru]
            [cmr.transmit.metadata-db :as mdb]
            [cmr.common.log :refer (debug info warn error)]
            [cmr.common.services.errors :as errors]
            [cmr.indexer.services.concepts.temporal :as temporal]
            [cmr.indexer.services.concepts.attribute :as attrib]
            [cmr.indexer.services.concepts.orbit-calculated-spatial-domain :as ocsd]
            [cmr.indexer.services.concepts.spatial :as spatial]
            [cmr.common.cache :as cache]))

(defmethod idx/parse-concept :granule
  [concept]
  (granule/parse-granule (:metadata concept)))

(defn- fetch-parent-collection
  "Retrieve the parent collection umm from the db"
  [context parent-collection-id]
  (let [parent-collection-cache (get-in context [:system :parent-collection-cache])
        concept (mdb/get-latest-concept context parent-collection-id)]
    (assoc (idx/parse-concept concept) :concept-id parent-collection-id)))

(defn- get-parent-collection
  [context parent-collection-id]
  (if-let [cache (get-in context [:system :parent-collection-cache])]
    (cache/cache-lookup cache parent-collection-id
                        (partial fetch-parent-collection context parent-collection-id))
    (fetch-parent-collection context parent-collection-id)))

(defn spatial->elastic
  [parent-collection granule]
  (when (get-in granule [:spatial-coverage :geometries])
    (let [gsr (get-in parent-collection [:spatial-coverage :granule-spatial-representation])]
      (if (= gsr :geodetic)
        (spatial/spatial->elastic-docs gsr granule)
        (warn
          "Only geodetic is supported for granule spatial representation currently. Ignoring this area.")))))

(defmethod idx/concept->elastic-doc :granule
  [context concept umm-granule]
  (let [{:keys [concept-id extra-fields provider-id revision-date]} concept
        {:keys [parent-collection-id]} extra-fields
        parent-collection (get-parent-collection context parent-collection-id)
        {:keys [granule-ur data-granule temporal platform-refs project-refs related-urls cloud-cover]} umm-granule
        {:keys [size producer-gran-id day-night]} data-granule
        platform-short-names (map :short-name platform-refs)
        instrument-refs (mapcat :instrument-refs platform-refs)
        instrument-short-names (remove nil? (map :short-name instrument-refs))
        sensor-refs (mapcat :sensor-refs instrument-refs)
        sensor-short-names (remove nil? (map :short-name sensor-refs))
        start-date (temporal/start-date :granule temporal)
        end-date (temporal/end-date :granule temporal)
        downloadable (not (empty? (filter ru/downloadable-url? related-urls)))]
    (merge {:concept-id concept-id
            :collection-concept-id parent-collection-id<commit_msg>Removing warning that was being logged way too many times.<commit_after>(ns cmr.indexer.services.concepts.granule
  "Contains functions to parse and convert granule concept"
  (:require [clojure.string :as s]
            [clj-time.format :as f]
            [cmr.indexer.services.index-service :as idx]
            [cmr.umm.echo10.granule :as granule]
            [cmr.umm.echo10.related-url :as ru]
            [cmr.transmit.metadata-db :as mdb]
            [cmr.common.log :refer (debug info warn error)]
            [cmr.common.services.errors :as errors]
            [cmr.indexer.services.concepts.temporal :as temporal]
            [cmr.indexer.services.concepts.attribute :as attrib]
            [cmr.indexer.services.concepts.orbit-calculated-spatial-domain :as ocsd]
            [cmr.indexer.services.concepts.spatial :as spatial]
            [cmr.common.cache :as cache]))

(defmethod idx/parse-concept :granule
  [concept]
  (granule/parse-granule (:metadata concept)))

(defn- fetch-parent-collection
  "Retrieve the parent collection umm from the db"
  [context parent-collection-id]
  (let [parent-collection-cache (get-in context [:system :parent-collection-cache])
        concept (mdb/get-latest-concept context parent-collection-id)]
    (assoc (idx/parse-concept concept) :concept-id parent-collection-id)))

(defn- get-parent-collection
  [context parent-collection-id]
  (if-let [cache (get-in context [:system :parent-collection-cache])]
    (cache/cache-lookup cache parent-collection-id
                        (partial fetch-parent-collection context parent-collection-id))
    (fetch-parent-collection context parent-collection-id)))

(defn spatial->elastic
  [parent-collection granule]
  (when (get-in granule [:spatial-coverage :geometries])
    (let [gsr (get-in parent-collection [:spatial-coverage :granule-spatial-representation])]
      (when (= gsr :geodetic)
        ;; Only index the types we support. We will silently ignore the other kinds. Adding a warning
        ;; turned out to be way too much logging.
        (spatial/spatial->elastic-docs gsr granule)))))

(defmethod idx/concept->elastic-doc :granule
  [context concept umm-granule]
  (let [{:keys [concept-id extra-fields provider-id revision-date]} concept
        {:keys [parent-collection-id]} extra-fields
        parent-collection (get-parent-collection context parent-collection-id)
        {:keys [granule-ur data-granule temporal platform-refs project-refs related-urls cloud-cover]} umm-granule
        {:keys [size producer-gran-id day-night]} data-granule
        platform-short-names (map :short-name platform-refs)
        instrument-refs (mapcat :instrument-refs platform-refs)
        instrument-short-names (remove nil? (map :short-name instrument-refs))
        sensor-refs (mapcat :sensor-refs instrument-refs)
        sensor-short-names (remove nil? (map :short-name sensor-refs))
        start-date (temporal/start-date :granule temporal)
        end-date (temporal/end-date :granule temporal)
        downloadable (not (empty? (filter ru/downloadable-url? related-urls)))]
    (merge {:concept-id concept-id
            :collection-concept-id parent-collection-id
{'fraction_non_alphanumeric': 0.11968804711125258, 'fraction_numerical': 0.001273277096928219, 'mean_word_length': 3.120655737704918, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1075986', 'n_tokens_mistral': 1995, 'n_tokens_neox': 1869, 'n_words': 455}