<commit_before>(ns cmr.ingest.services.ingest
  (:require [clj-time.core :as t]
            [cmr.transmit.metadata-db :as mdb]
            [cmr.ingest.data.indexer :as indexer]
            [cmr.ingest.services.messages :as msg]
            [cmr.ingest.services.validation :as v]
            [cmr.common.log :refer (debug info warn error)]
            [cmr.common.services.errors :as serv-errors]
            [cmr.common.services.messages :as cmsg]
            [cmr.common.date-time-parser :as p]
            [cmr.umm.echo10.collection :as c]
            [cmr.umm.echo10.granule :as g]
            [clojure.string :as string]
            [cmr.system-trace.core :refer [deftracefn]]))

(defmulti add-extra-fields
  "Parse the metadata of concept, add the extra fields to it and return the concept."
  (fn [context concept]
    (:concept-type concept)))

(defmethod add-extra-fields :collection
  [context concept]
  (let [collection (c/parse-collection (:metadata concept))
        {{:keys [short-name version-id]} :product
         {:keys [delete-time]} :data-provider-timestamps
         entry-title :entry-title} collection]
    (assoc concept :extra-fields {:entry-title entry-title
                                  :short-name short-name
                                  :version-id version-id
                                  :delete-time (when delete-time (str delete-time))})))

(defmethod add-extra-fields :granule
  [context concept]
  (let [granule (g/parse-granule (:metadata concept))
        {:keys [collection-ref granule-ur]
         {:keys [delete-time]} :data-provider-timestamps} granule
        params (merge {:provider-id (:provider-id concept)} collection-ref)
        params (into {} (remove (comp empty? second) params))
        parent-collection-id (mdb/get-collection-concept-id context params)]
    (when-not parent-collection-id
      (cmsg/data-error :not-found
                       msg/parent-collection-does-not-exist
                       granule-ur))
    (assoc concept :extra-fields {:parent-collection-id parent-collection-id
                                  :delete-time (when delete-time (str delete-time))})))

(deftracefn save-concept
  "Store a concept in mdb and indexer and return concept-id and revision-id."
  [context concept]
  (v/validate concept)
  (let [concept (add-extra-fields context concept)
        time-to-compare (t/plus (t/now) (t/minutes 1))
        delete-time (get-in concept [:extra-fields :delete-time])
        delete-time (if delete-time (p/parse-datetime delete-time) nil)]
    (if (and delete-time (t/after? time-to-compare delete-time))
      (serv-errors/throw-service-error
        :bad-request
        (format "DeleteTime %s is before the current time." (str delete-time)))
      (let [{:keys [concept-id revision-id]} (mdb/save-concept context concept)]
        (indexer/index-concept context concept-id revision-id)
        {:concept-id concept-id, :revision-id revision-id}))))

(deftracefn delete-concept
  "Delete a concept from mdb and indexer."<commit_msg>Updating the ingest app to use the umm.core namespace.<commit_after>(ns cmr.ingest.services.ingest
  (:require [clj-time.core :as t]
            [cmr.transmit.metadata-db :as mdb]
            [cmr.ingest.data.indexer :as indexer]
            [cmr.ingest.services.messages :as msg]
            [cmr.ingest.services.validation :as v]
            [cmr.common.log :refer (debug info warn error)]
            [cmr.common.services.errors :as serv-errors]
            [cmr.common.services.messages :as cmsg]
            [cmr.common.date-time-parser :as p]
            [cmr.umm.core :as umm]
            [clojure.string :as string]
            [cmr.system-trace.core :refer [deftracefn]]))

(defmulti add-extra-fields
  "Parse the metadata of concept, add the extra fields to it and return the concept."
  (fn [context concept]
    (:concept-type concept)))

(defmethod add-extra-fields :collection
  [context concept]
  (let [collection (umm/parse-concept concept)
        {{:keys [short-name version-id]} :product
         {:keys [delete-time]} :data-provider-timestamps
         entry-title :entry-title} collection]
    (assoc concept :extra-fields {:entry-title entry-title
                                  :short-name short-name
                                  :version-id version-id
                                  :delete-time (when delete-time (str delete-time))})))

(defmethod add-extra-fields :granule
  [context concept]
  (let [granule (umm/parse-concept concept)
        {:keys [collection-ref granule-ur]
         {:keys [delete-time]} :data-provider-timestamps} granule
        params (merge {:provider-id (:provider-id concept)} collection-ref)
        params (into {} (remove (comp empty? second) params))
        parent-collection-id (mdb/get-collection-concept-id context params)]
    (when-not parent-collection-id
      (cmsg/data-error :not-found
                       msg/parent-collection-does-not-exist
                       granule-ur))
    (assoc concept :extra-fields {:parent-collection-id parent-collection-id
                                  :delete-time (when delete-time (str delete-time))})))

(deftracefn save-concept
  "Store a concept in mdb and indexer and return concept-id and revision-id."
  [context concept]
  (v/validate concept)
  (let [concept (add-extra-fields context concept)
        time-to-compare (t/plus (t/now) (t/minutes 1))
        delete-time (get-in concept [:extra-fields :delete-time])
        delete-time (if delete-time (p/parse-datetime delete-time) nil)]
    (if (and delete-time (t/after? time-to-compare delete-time))
      (serv-errors/throw-service-error
        :bad-request
        (format "DeleteTime %s is before the current time." (str delete-time)))
      (let [{:keys [concept-id revision-id]} (mdb/save-concept context concept)]
        (indexer/index-concept context concept-id revision-id)
        {:concept-id concept-id, :revision-id revision-id}))))

(deftracefn delete-concept
  "Delete a concept from mdb and indexer."
{'fraction_non_alphanumeric': 0.12495862297252565, 'fraction_numerical': 0.0009930486593843098, 'mean_word_length': 2.383538633818589, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6325666', 'n_tokens_mistral': 1844, 'n_tokens_neox': 1795, 'n_words': 466}