<commit_before>(ns cmr.indexer.data.concepts.collection
  "Contains functions to parse and convert collection concept"
  (:require [clojure.string :as str]
            [clj-time.format :as f]
            [cheshire.core :as json]
            [camel-snake-kebab :as csk]
            [cmr.indexer.services.index-service :as idx]
            [cmr.indexer.data.elasticsearch :as es]
            [cmr.common.mime-types :as mt]
            [cmr.common.log :refer (debug info warn error)]
            [cmr.common.services.errors :as errors]
            [cmr.umm.related-url-helper :as ru]
            [cmr.umm.start-end-date :as sed]
            [cmr.indexer.data.concepts.attribute :as attrib]
            [cmr.indexer.data.concepts.science-keyword :as sk]
            [cmr.indexer.data.concepts.spatial :as spatial]
            [cmr.indexer.data.concepts.keyword :as k]
            [cmr.indexer.data.concepts.organization :as org]
            [cmr.acl.core :as acl]
            [cmr.common.concepts :as concepts])
  (:import cmr.spatial.mbr.Mbr))

(defn spatial->elastic
  [collection]
  (when-let [geometries (seq (get-in collection [:spatial-coverage :geometries]))]
    (let [sr (get-in collection [:spatial-coverage :spatial-representation])]
      (cond
        (or (= sr :geodetic) (= sr :cartesian))
        (spatial/spatial->elastic-docs sr collection)

        :else
        (errors/internal-error! (str "Unknown spatial representation [" sr "]"))))))

(defn- person->email-contact
  "Return an email contact for the Personnel record or nil if none is available."
  [person]
  (some (fn [contact]
          (= :email
             (:type contact)))
        (:contacts person)))

(defn person-with-email
  "Returns the first Personnel record for the list with an email contact or
  nil if none exists."
  [personnel]
  (some (fn [person]
          (person->email-contact person))
        personnel))

(defmethod es/concept->elastic-doc :collection
  [context concept collection]
  (let [{:keys [concept-id provider-id revision-date format]} concept
        {{:keys [short-name long-name version-id processing-level-id collection-data-type]} :product
         :keys [entry-id entry-title summary temporal related-urls spatial-keywords associated-difs
                temporal-keywords access-value personnel distribution]} collection
        collection-data-type (if (= "NEAR_REAL_TIME" collection-data-type)
                               ;; add in all the aliases for NEAR_REAL_TIME
                               (concat [collection-data-type] k/nrt-aliases)
                               collection-data-type)
        personnel (person-with-email personnel)
        platforms (:platforms collection)
        platform-short-names (map :short-name platforms)
        platform-long-names (remove nil? (map :long-name platforms))
        instruments (mapcat :instruments platforms)
        instrument-short-names (remove nil? (map :short-name instruments))
        instrument-long-names (remove nil? (map :long-name instruments))
        sensors (mapcat :sensors instruments)
        sensor-short-names (remove nil? (map :short-name sensors))
        sensor-long-names (remove nil? (map :long-name sensors))
        project-short-names (map :short-name (:projects collection))
        project-long-names (remove nil? (map :long-name (:projects collection)))
        two-d-coord-names (map :name (:two-d-coordinate-systems collection))
        archive-center-val (org/extract-archive-centers collection)
        start-date (sed/start-date :collection temporal)
        end-date (sed/end-date :collection temporal)
        atom-links (map json/generate-string (ru/atom-links related-urls))
        ;; not empty is used below to get a real true/false value
        downloadable (not (empty? (ru/downloadable-urls related-urls)))
        browsable (not (empty? (ru/browse-urls related-urls)))
        update-time (get-in collection [:data-provider-timestamps :update-time])
        update-time (f/unparse (f/formatters :date-time) update-time)
        insert-time (get-in collection [:data-provider-timestamps :insert-time])
        insert-time (f/unparse (f/formatters :date-time) insert-time)
        spatial-representation (get-in collection [:spatial-coverage :spatial-representation])
        permitted-group-ids (acl/get-coll-permitted-group-ids context provider-id collection)]
    (merge {:concept-id concept-id
            :concept-seq-id (:sequence-number (concepts/parse-concept-id concept-id))
            :permitted-group-ids permitted-group-ids
            :entry-id entry-id
            :entry-id.lowercase (str/lower-case entry-id)
            :entry-title entry-title
            :entry-title.lowercase (str/lower-case entry-title)
            :provider-id provider-id
            :provider-id.lowercase (str/lower-case provider-id)
            :short-name short-name
            :short-name.lowercase (when short-name (str/lower-case short-name))
            :version-id version-id
            :version-id.lowercase (when version-id (str/lower-case version-id))
            :revision-date revision-date
            :access-value access-value
            :processing-level-id processing-level-id
            :processing-level-id.lowercase (when processing-level-id (str/lower-case processing-level-id))
            :collection-data-type collection-data-type
            :collection-data-type.lowercase (when collection-data-type
                                              (if (sequential? collection-data-type)
                                                (map str/lower-case collection-data-type)
                                                (str/lower-case collection-data-type)))
            :platform-sn platform-short-names
            :platform-sn.lowercase  (map str/lower-case platform-short-names)
            :instrument-sn instrument-short-names
            :instrument-sn.lowercase  (map str/lower-case instrument-short-names)
            :sensor-sn sensor-short-names
            :sensor-sn.lowercase  (map str/lower-case sensor-short-names)
            :project-sn project-short-names
            :project-sn.lowercase  (map str/lower-case project-short-names)
            :two-d-coord-name two-d-coord-names
            :two-d-coord-name.lowercase  (map str/lower-case two-d-coord-names)
            :spatial-keyword spatial-keywords
            :spatial-keyword.lowercase  (map str/lower-case spatial-keywords)
            :attributes (attrib/psas->elastic-docs collection)
            :science-keywords (sk/science-keywords->elastic-doc collection)
            :science-keywords-flat (sk/flatten-science-keywords collection)
            :personnel (map json/generate-string personnel)
            :start-date (when start-date (f/unparse (f/formatters :date-time) start-date))
            :end-date (when end-date (f/unparse (f/formatters :date-time) end-date))
            :archive-center archive-center-val
            :archive-center.lowercase (map str/lower-case archive-center-val)
            :downloadable downloadable
            :browsable browsable
            :atom-links atom-links
            :summary summary
            :metadata-format (name (mt/base-mime-type-to-format format))
            :related-urls (map json/generate-string related-urls)
            :update-time update-time
            :insert-time insert-time
            :associated-difs associated-difs
            :associated-difs.lowercase (map str/lower-case associated-difs)
            :coordinate-system (when spatial-representation (csk/->SNAKE_CASE_STRING spatial-representation))
            ;; fields added to support keyword searches
            :keyword (k/create-keywords-field concept-id collection)
            :long-name.lowercase (when long-name (str/lower-case long-name))
            :platform-ln.lowercase (map str/lower-case platform-long-names)
            :instrument-ln.lowercase (map str/lower-case instrument-long-names)
            :sensor-ln.lowercase (map str/lower-case sensor-long-names)
            :project-ln.lowercase (map str/lower-case project-long-names)
            :temporal-keyword.lowercase (map str/lower-case temporal-keywords)}
           (get-in collection [:spatial-coverage :orbit-parameters])
           (spatial->elastic collection)
           (sk/science-keywords->facet-fields collection))))<commit_msg>Making temporary quick fix to the indexer app to fix a problem with collection indexing.<commit_after>(ns cmr.indexer.data.concepts.collection
  "Contains functions to parse and convert collection concept"
  (:require [clojure.string :as str]
            [clj-time.format :as f]
            [cheshire.core :as json]
            [camel-snake-kebab :as csk]
            [cmr.indexer.services.index-service :as idx]
            [cmr.indexer.data.elasticsearch :as es]
            [cmr.common.mime-types :as mt]
            [cmr.common.log :refer (debug info warn error)]
            [cmr.common.services.errors :as errors]
            [cmr.umm.related-url-helper :as ru]
            [cmr.umm.start-end-date :as sed]
            [cmr.indexer.data.concepts.attribute :as attrib]
            [cmr.indexer.data.concepts.science-keyword :as sk]
            [cmr.indexer.data.concepts.spatial :as spatial]
            [cmr.indexer.data.concepts.keyword :as k]
            [cmr.indexer.data.concepts.organization :as org]
            [cmr.acl.core :as acl]
            [cmr.common.concepts :as concepts])
  (:import cmr.spatial.mbr.Mbr))

(defn spatial->elastic
  [collection]
  (when-let [geometries (seq (get-in collection [:spatial-coverage :geometries]))]
    (let [sr (get-in collection [:spatial-coverage :spatial-representation])]
      (cond
        (or (= sr :geodetic) (= sr :cartesian))
        (spatial/spatial->elastic-docs sr collection)

        :else
        (errors/internal-error! (str "Unknown spatial representation [" sr "]"))))))

(defn- person->email-contact
  "Return an email contact for the Personnel record or nil if none is available."
  [person]
  (first (filter (fn [contact]
                   (= :email
                      (:type contact)))
                 (:contacts person))))

(defn person-with-email
  "Returns the first Personnel record for the list with an email contact or
  nil if none exists."
  [personnel]
  (first (filter person->email-contact personnel)))

(defmethod es/concept->elastic-doc :collection
  [context concept collection]
  (let [{:keys [concept-id provider-id revision-date format]} concept
        {{:keys [short-name long-name version-id processing-level-id collection-data-type]} :product
         :keys [entry-id entry-title summary temporal related-urls spatial-keywords associated-difs
                temporal-keywords access-value personnel distribution]} collection
        collection-data-type (if (= "NEAR_REAL_TIME" collection-data-type)
                               ;; add in all the aliases for NEAR_REAL_TIME
                               (concat [collection-data-type] k/nrt-aliases)
                               collection-data-type)
        personnel (person-with-email personnel)
        platforms (:platforms collection)
        platform-short-names (map :short-name platforms)
        platform-long-names (remove nil? (map :long-name platforms))
        instruments (mapcat :instruments platforms)
        instrument-short-names (remove nil? (map :short-name instruments))
        instrument-long-names (remove nil? (map :long-name instruments))
        sensors (mapcat :sensors instruments)
        sensor-short-names (remove nil? (map :short-name sensors))
        sensor-long-names (remove nil? (map :long-name sensors))
        project-short-names (map :short-name (:projects collection))
        project-long-names (remove nil? (map :long-name (:projects collection)))
        two-d-coord-names (map :name (:two-d-coordinate-systems collection))
        archive-center-val (org/extract-archive-centers collection)
        start-date (sed/start-date :collection temporal)
        end-date (sed/end-date :collection temporal)
        atom-links (map json/generate-string (ru/atom-links related-urls))
        ;; not empty is used below to get a real true/false value
        downloadable (not (empty? (ru/downloadable-urls related-urls)))
        browsable (not (empty? (ru/browse-urls related-urls)))
        update-time (get-in collection [:data-provider-timestamps :update-time])
        update-time (f/unparse (f/formatters :date-time) update-time)
        insert-time (get-in collection [:data-provider-timestamps :insert-time])
        insert-time (f/unparse (f/formatters :date-time) insert-time)
        spatial-representation (get-in collection [:spatial-coverage :spatial-representation])
        permitted-group-ids (acl/get-coll-permitted-group-ids context provider-id collection)]
    (merge {:concept-id concept-id
            :concept-seq-id (:sequence-number (concepts/parse-concept-id concept-id))
            :permitted-group-ids permitted-group-ids
            :entry-id entry-id
            :entry-id.lowercase (str/lower-case entry-id)
            :entry-title entry-title
            :entry-title.lowercase (str/lower-case entry-title)
            :provider-id provider-id
            :provider-id.lowercase (str/lower-case provider-id)
            :short-name short-name
            :short-name.lowercase (when short-name (str/lower-case short-name))
            :version-id version-id
            :version-id.lowercase (when version-id (str/lower-case version-id))
            :revision-date revision-date
            :access-value access-value
            :processing-level-id processing-level-id
            :processing-level-id.lowercase (when processing-level-id (str/lower-case processing-level-id))
            :collection-data-type collection-data-type
            :collection-data-type.lowercase (when collection-data-type
                                              (if (sequential? collection-data-type)
                                                (map str/lower-case collection-data-type)
                                                (str/lower-case collection-data-type)))
            :platform-sn platform-short-names
            :platform-sn.lowercase  (map str/lower-case platform-short-names)
            :instrument-sn instrument-short-names
            :instrument-sn.lowercase  (map str/lower-case instrument-short-names)
            :sensor-sn sensor-short-names
            :sensor-sn.lowercase  (map str/lower-case sensor-short-names)
            :project-sn project-short-names
            :project-sn.lowercase  (map str/lower-case project-short-names)
            :two-d-coord-name two-d-coord-names
            :two-d-coord-name.lowercase  (map str/lower-case two-d-coord-names)
            :spatial-keyword spatial-keywords
            :spatial-keyword.lowercase  (map str/lower-case spatial-keywords)
            :attributes (attrib/psas->elastic-docs collection)
            :science-keywords (sk/science-keywords->elastic-doc collection)
            :science-keywords-flat (sk/flatten-science-keywords collection)
            :personnel (json/generate-string personnel)
            :start-date (when start-date (f/unparse (f/formatters :date-time) start-date))
            :end-date (when end-date (f/unparse (f/formatters :date-time) end-date))
            :archive-center archive-center-val
            :archive-center.lowercase (map str/lower-case archive-center-val)
            :downloadable downloadable
            :browsable browsable
            :atom-links atom-links
            :summary summary
            :metadata-format (name (mt/base-mime-type-to-format format))
            :related-urls (map json/generate-string related-urls)
            :update-time update-time
            :insert-time insert-time
            :associated-difs associated-difs
            :associated-difs.lowercase (map str/lower-case associated-difs)
            :coordinate-system (when spatial-representation (csk/->SNAKE_CASE_STRING spatial-representation))
            ;; fields added to support keyword searches
            :keyword (k/create-keywords-field concept-id collection)
            :long-name.lowercase (when long-name (str/lower-case long-name))
            :platform-ln.lowercase (map str/lower-case platform-long-names)
            :instrument-ln.lowercase (map str/lower-case instrument-long-names)
            :sensor-ln.lowercase (map str/lower-case sensor-long-names)
            :project-ln.lowercase (map str/lower-case project-long-names)
            :temporal-keyword.lowercase (map str/lower-case temporal-keywords)}
           (get-in collection [:spatial-coverage :orbit-parameters])
           (spatial->elastic collection)
           (sk/science-keywords->facet-fields collection))))
{'fraction_non_alphanumeric': 0.11337019202187741, 'fraction_numerical': 0.0, 'mean_word_length': 2.2964922594552224, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4889266', 'n_tokens_mistral': 4811, 'n_tokens_neox': 4673, 'n_words': 1087}