<commit_before>(ns cmr.ingest.services.ingest-service
  (:require
    [cmr.common.cache :as cache]
    [cmr.common.config :as cfg :refer [defconfig]]
    [cmr.common.log :refer (debug info warn error)]
    [cmr.common.services.messages :as cmsg]
    [cmr.common.services.errors :as errors]
    [cmr.common.util :as util :refer [defn-timed]]
    [cmr.ingest.config :as config]
    [cmr.ingest.data.ingest-events :as ingest-events]
    [cmr.ingest.data.provider-acl-hash :as pah]
    [cmr.ingest.services.messages :as msg]
    [cmr.ingest.services.helper :as h]
    [cmr.ingest.validation.validation :as v]
    [cmr.message-queue.services.queue :as queue]
    [cmr.oracle.connection :as conn]
    [cmr.transmit.cubby :as cubby]
    [cmr.transmit.echo.rest :as rest]
    [cmr.transmit.indexer :as indexer]
    [cmr.transmit.metadata-db :as mdb]
    [cmr.transmit.metadata-db2 :as mdb2]
    [cmr.umm.collection.entry-id :as eid]
    [cmr.umm-spec.legacy :as umm-legacy]
    [cmr.umm-spec.umm-spec-core :as spec]
    [cmr.umm-spec.versioning :as ver]))

(defn add-extra-fields-for-collection
  "Returns collection concept with fields necessary for ingest into metadata db
  under :extra-fields."
  [context concept collection]
  (let [{short-name :ShortName
         version-id :Version
         entry-title :EntryTitle} collection
        entry-id (eid/entry-id short-name version-id)
        delete-time (first (map :Date (filter #(= "DELETE" (:Type %)) (:DataDates collection))))]
    (assoc concept :extra-fields {:entry-title entry-title
                                  :entry-id entry-id
                                  :short-name short-name
                                  :version-id version-id
                                  :delete-time (when delete-time (str delete-time))})))

(defn- validate-and-parse-collection-concept
  "Validates a collection concept and parses it. Returns the UMM record and any warnings from
  validation."
  [context collection-concept validation-options]
  (v/validate-concept-request collection-concept)
  (v/validate-concept-metadata collection-concept)
  (let [{:keys [format metadata]} collection-concept
        collection (spec/parse-metadata context :collection format metadata {:sanitize? false})

    ;; Validate against the UMM Spec validation rules
        warnings (v/validate-collection-umm-spec context collection validation-options)]
    ;; Using the legacy UMM validation rules (for now)
    (v/validate-collection-umm context
                               (umm-legacy/parse-concept context collection-concept)
                               (:validate-keywords? validation-options))
    ;; The UMM Spec collection is returned
    {:collection collection
     :warnings warnings}))

(defn-timed validate-and-prepare-collection
  "Validates the collection and adds extra fields needed for metadata db. Throws a service error
  if any validation issues are found and errors are enabled, otherwise returns errors as warnings."
  [context concept validation-options]
  (let [concept (update-in concept [:format] ver/fix-concept-format)<commit_msg>CMR-2112 Fix business rule validation fail<commit_after>(ns cmr.ingest.services.ingest-service
  (:require
    [cmr.common.cache :as cache]
    [cmr.common.config :as cfg :refer [defconfig]]
    [cmr.common.log :refer (debug info warn error)]
    [cmr.common.services.messages :as cmsg]
    [cmr.common.services.errors :as errors]
    [cmr.common.util :as util :refer [defn-timed]]
    [cmr.ingest.config :as config]
    [cmr.ingest.data.ingest-events :as ingest-events]
    [cmr.ingest.data.provider-acl-hash :as pah]
    [cmr.ingest.services.messages :as msg]
    [cmr.ingest.services.helper :as h]
    [cmr.ingest.validation.validation :as v]
    [cmr.message-queue.services.queue :as queue]
    [cmr.oracle.connection :as conn]
    [cmr.transmit.cubby :as cubby]
    [cmr.transmit.echo.rest :as rest]
    [cmr.transmit.indexer :as indexer]
    [cmr.transmit.metadata-db :as mdb]
    [cmr.transmit.metadata-db2 :as mdb2]
    [cmr.umm.collection.entry-id :as eid]
    [cmr.umm-spec.legacy :as umm-legacy]
    [cmr.umm-spec.umm-spec-core :as spec]
    [cmr.umm-spec.versioning :as ver]))

(defn add-extra-fields-for-collection
  "Returns collection concept with fields necessary for ingest into metadata db
  under :extra-fields."
  [context concept collection]
  (let [{short-name :ShortName
         version-id :Version
         entry-title :EntryTitle} collection
        entry-id (eid/entry-id short-name version-id)
        delete-time (first (map :Date (filter #(= "DELETE" (:Type %)) (:DataDates collection))))]
    (assoc concept :extra-fields {:entry-title entry-title
                                  :entry-id entry-id
                                  :short-name short-name
                                  :version-id version-id
                                  :delete-time (when delete-time (str delete-time))})))

(defn- validate-and-parse-collection-concept
  "Validates a collection concept and parses it. Returns the UMM record and any warnings from
  validation."
  [context collection-concept validation-options]
  (v/validate-concept-request collection-concept)
  (v/validate-concept-metadata collection-concept)
  (let [{:keys [format metadata]} collection-concept
        collection (spec/parse-metadata context :collection format metadata {:sanitize? false})
        sanitized-collection (spec/parse-metadata context :collection format metadata {:sanitize? true})

    ;; Validate against the UMM Spec validation rules
        warnings (v/validate-collection-umm-spec context collection validation-options)]
    ;; Using the legacy UMM validation rules (for now)
    (v/validate-collection-umm context
                               (umm-legacy/parse-concept context collection-concept)
                               (:validate-keywords? validation-options))
    ;; The UMM Spec collection is returned
    {:collection sanitized-collection
     :warnings warnings}))

(defn-timed validate-and-prepare-collection
  "Validates the collection and adds extra fields needed for metadata db. Throws a service error
  if any validation issues are found and errors are enabled, otherwise returns errors as warnings."
  [context concept validation-options]
  (let [concept (update-in concept [:format] ver/fix-concept-format)
{'fraction_non_alphanumeric': 0.1105511811023622, 'fraction_numerical': 0.0012598425196850393, 'mean_word_length': 2.8844036697247706, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6341095', 'n_tokens_mistral': 1923, 'n_tokens_neox': 1876, 'n_words': 512}