<commit_before>(ns cmr.ingest.api.subscriptions
  "Subscription ingest functions in support of the ingest API."
  (:require
   [camel-snake-kebab.core :as csk]
   [cheshire.core :as json]
   [clojure.string :as string]
   [cmr.acl.core :as acl]
   [cmr.common-app.api.enabled :as common-enabled]
   [cmr.common-app.api.launchpad-token-validation :as lt-validation]
   [cmr.common-app.services.ingest.subscription-common :as sub-common]
   [cmr.common.log :refer [debug info warn error]]
   [cmr.common.services.errors :as errors]
   [cmr.ingest.api.core :as api-core]
   [cmr.ingest.services.ingest-service :as ingest]
   [cmr.ingest.services.subscriptions-helper :as jobs]
   [cmr.ingest.validation.validation :as v]
   [cmr.transmit.access-control :as access-control]
   [cmr.transmit.config :as config]
   [cmr.transmit.metadata-db :as mdb]
   [cmr.transmit.metadata-db2 :as mdb2]
   [cmr.transmit.search :as search]
   [cmr.transmit.urs :as urs])
  (:import
   [java.util UUID]))

(defn- validate-and-prepare-subscription-concept
  "Validate subscription concept, set the concept format and returns the concept;
  throws error if the metadata is not a valid against the UMM subscription JSON schema."
  [concept]
  (let [concept (update-in concept [:format] (partial ingest/fix-ingest-concept-format :subscription))]
    (v/validate-concept-request concept)
    (v/validate-concept-metadata concept)
    concept))

(defn- subscriber-collection-permission-error
  [subscriber-id concept-id]
  (errors/throw-service-error
   :unauthorized
   (format "Collection with concept id [%s] does not exist or subscriber-id [%s] does not have permission to view the collection."
           concept-id
           subscriber-id)))

(defn- check-subscriber-collection-permission
  "Checks that the subscriber-id can read the collection supplied in the subscription metadata"
  [request-context concept]
  (let [metadata (-> (:metadata concept)
                     (json/decode true))
        concept-id (:CollectionConceptId metadata)
        subscriber-id (:SubscriberId metadata)]
      (try
        (let [permissions (-> (access-control/get-permissions request-context
                                                              {:concept_id concept-id
                                                               :user_id subscriber-id})
                              json/decode
                              (get concept-id))]
          (when-not (some #{"read"} permissions)
            (subscriber-collection-permission-error
             subscriber-id
             concept-id)))
        (catch Exception e
          (subscriber-collection-permission-error
           subscriber-id
           concept-id)))))

(defn- search-gran-refs-with-sub-params
  "Performs a granule search using the provided query parameters. If the query is no good,
   we throw a service error."
  [context params]
  (when-let [errors (search/validate-granule-search-params context params)]
    (errors/throw-service-error :bad-request
      (str "Subscription query validation failed with the following error(s): " errors))))

(defn- validate-query
  "Performs a granule search using subscription query parameters for purposes of validation"
  [request-context subscription]
  (let [metadata (-> (:metadata subscription) (json/decode true))
        collection-id (:CollectionConceptId metadata)
        query-string (:Query metadata)
        query-params (jobs/create-query-params query-string)
        search-params (merge {:collection-concept-id collection-id
                              :token (config/echo-system-token)}
                             query-params)
        gran-refs (search-gran-refs-with-sub-params request-context search-params)]))

(defn- check-duplicate-subscription
  "The query used by a subscriber for a collection should be unique to prevent
   redundent emails from being sent to them. This function will check that a
   subscription is unique for the following conditions: native-id, collection-id,
   subscriber-id, normalized-query."
  [request-context subscription]
  (let [native-id (:native-id subscription)
        provider-id (:provider-id subscription)
        metadata (-> (:metadata subscription) (json/decode true))
        normalized-query (:normalized-query subscription)
        collection-id (:CollectionConceptId metadata)
        subscriber-id (:SubscriberId metadata)
        ;; Find concepts with matching collection-concept-id, normalized-query, and subscriber-id
        duplicate-queries (mdb/find-concepts
                           request-context
                           {:collection-concept-id collection-id
                            :normalized-query normalized-query
                            :subscriber-id subscriber-id
                            :exclude-metadata true
                            :latest true}
                           :subscription)
        ;;we only want to look at non-deleted subscriptions
        active-duplicate-queries (remove :deleted duplicate-queries)]
     ;;If there is at least one duplicate subscription,
     ;;We need to make sure it has the same native-id, or else reject the ingest
     (when (and (> (count active-duplicate-queries) 0)
                (every? #(not= native-id (:native-id %)) active-duplicate-queries))
       (errors/throw-service-error
        :conflict
        (format (str "The subscriber-id [%s] has already subscribed to the "
                     "collection with concept-id [%s] using the query [%s]. "
                     "Subscribers must use unique queries for each Collection.")
                subscriber-id collection-id  normalized-query)))))

(defn- get-subscriber-id
  "Returns the subscriber id of the given subscription concept by parsing its metadata."
  [sub-concept]
  (-> sub-concept
      :metadata
      (json/decode true)
      :SubscriberId))

(defn- perform-subscription-ingest
  "This function assumes all checks have already taken place and that a
  subscription is ready to be saved"
  [request-context concept headers]
  (let [validated-concept (validate-and-prepare-subscription-concept concept)
        _ (check-subscriber-collection-permission request-context concept)
        concept-with-user-id (api-core/set-user-id validated-concept
                                                   request-context
                                                   headers)
        ;; Log the ingest attempt before the save
        _ (info (format "Ingesting subscription %s from client %s"
                        (api-core/concept->loggable-string concept-with-user-id)
                        (:client-id request-context)))
        save-subscription-result (ingest/save-subscription request-context
                                                           concept-with-user-id)]
    ;; Log the successful ingest, with the metadata size in bytes.
    (api-core/log-concept-with-metadata-size concept-with-user-id request-context)
    (api-core/generate-ingest-response headers save-subscription-result)))

(defn- common-ingest-checks
  "Common checks needed before starting to process an ingest operation"
  [request-context provider-id]
  (common-enabled/validate-write-enabled request-context "ingest")
  (lt-validation/validate-launchpad-token request-context)
  (api-core/verify-provider-exists request-context provider-id))

(defn- check-ingest-permission
  "Raise error if the user in the given context does not have the necessary permission to
   create/update/delete a subscription based on the new and old subscriber and existing ACLs."
  ([context provider-id new-subscriber]
   (check-ingest-permission context provider-id new-subscriber nil))
  ([context provider-id new-subscriber old-subscriber]
   (let [token-user (api-core/get-user-id-from-token context)]
     (if (and token-user
              (or (= token-user new-subscriber old-subscriber)
                  (and (= token-user new-subscriber)
                       (nil? old-subscriber))))
       (info (format "ACLs were bypassed because token user matches the subscriber [%s]."
                     new-subscriber))
       (do
         ;; log ACL checking message
         (if (and old-subscriber
                  (not= new-subscriber old-subscriber))
           (info (format "ACLs were checked because subscription subscriber is changed from [%s] to [%s]."
                         old-subscriber
                         new-subscriber))
           (info (format (str "ACLs were checked because token user [%s] is different from "
                              "the subscriber [%s] in the metadata.")
                         token-user
                         new-subscriber)))

         (acl/verify-ingest-management-permission
           context :update :provider-object provider-id)
         (acl/verify-subscription-management-permission
           context :update :provider-object provider-id))))))

(defn check-valid-user
  "Raise error if the user provided does not exist"
  [context user-id]
  (when (string/blank? user-id)
    (errors/throw-service-error
     :bad-request
     "INGEST FAILED - No ID was provided. Please provide a SubscriberId or pass in a valid token."))
  (when-not (urs/user-exists? context user-id) 
    (errors/throw-service-error
     :bad-request
     (format "The user-id [%s] must correspond to a valid EDL account." 
             user-id))))

(defn generate-native-id
  "Generate a native-id for a subscription based on the name."
  [subscription]
  (-> subscription
      :metadata
      (json/parse-string true)
      :Name
      csk/->snake_case
      (str "_" (UUID/randomUUID))))

(defn native-id-collision?
  "Queries metadata db for a matching provider-id and native-id pair."
  [context provider-id native-id]
  (let [query {:provider-id provider-id<commit_msg>CMR-7197: Updated error messages for when check-valid-user fails<commit_after>(ns cmr.ingest.api.subscriptions
  "Subscription ingest functions in support of the ingest API."
  (:require
   [camel-snake-kebab.core :as csk]
   [cheshire.core :as json]
   [clojure.string :as string]
   [cmr.acl.core :as acl]
   [cmr.common-app.api.enabled :as common-enabled]
   [cmr.common-app.api.launchpad-token-validation :as lt-validation]
   [cmr.common-app.services.ingest.subscription-common :as sub-common]
   [cmr.common.log :refer [debug info warn error]]
   [cmr.common.services.errors :as errors]
   [cmr.ingest.api.core :as api-core]
   [cmr.ingest.services.ingest-service :as ingest]
   [cmr.ingest.services.subscriptions-helper :as jobs]
   [cmr.ingest.validation.validation :as v]
   [cmr.transmit.access-control :as access-control]
   [cmr.transmit.config :as config]
   [cmr.transmit.metadata-db :as mdb]
   [cmr.transmit.metadata-db2 :as mdb2]
   [cmr.transmit.search :as search]
   [cmr.transmit.urs :as urs])
  (:import
   [java.util UUID]))

(defn- validate-and-prepare-subscription-concept
  "Validate subscription concept, set the concept format and returns the concept;
  throws error if the metadata is not a valid against the UMM subscription JSON schema."
  [concept]
  (let [concept (update-in concept [:format] (partial ingest/fix-ingest-concept-format :subscription))]
    (v/validate-concept-request concept)
    (v/validate-concept-metadata concept)
    concept))

(defn- subscriber-collection-permission-error
  [subscriber-id concept-id]
  (errors/throw-service-error
   :unauthorized
   (format "Collection with concept id [%s] does not exist or subscriber-id [%s] does not have permission to view the collection."
           concept-id
           subscriber-id)))

(defn- check-subscriber-collection-permission
  "Checks that the subscriber-id can read the collection supplied in the subscription metadata"
  [request-context concept]
  (let [metadata (-> (:metadata concept)
                     (json/decode true))
        concept-id (:CollectionConceptId metadata)
        subscriber-id (:SubscriberId metadata)]
      (try
        (let [permissions (-> (access-control/get-permissions request-context
                                                              {:concept_id concept-id
                                                               :user_id subscriber-id})
                              json/decode
                              (get concept-id))]
          (when-not (some #{"read"} permissions)
            (subscriber-collection-permission-error
             subscriber-id
             concept-id)))
        (catch Exception e
          (subscriber-collection-permission-error
           subscriber-id
           concept-id)))))

(defn- search-gran-refs-with-sub-params
  "Performs a granule search using the provided query parameters. If the query is no good,
   we throw a service error."
  [context params]
  (when-let [errors (search/validate-granule-search-params context params)]
    (errors/throw-service-error :bad-request
      (str "Subscription query validation failed with the following error(s): " errors))))

(defn- validate-query
  "Performs a granule search using subscription query parameters for purposes of validation"
  [request-context subscription]
  (let [metadata (-> (:metadata subscription) (json/decode true))
        collection-id (:CollectionConceptId metadata)
        query-string (:Query metadata)
        query-params (jobs/create-query-params query-string)
        search-params (merge {:collection-concept-id collection-id
                              :token (config/echo-system-token)}
                             query-params)
        gran-refs (search-gran-refs-with-sub-params request-context search-params)]))

(defn- check-duplicate-subscription
  "The query used by a subscriber for a collection should be unique to prevent
   redundent emails from being sent to them. This function will check that a
   subscription is unique for the following conditions: native-id, collection-id,
   subscriber-id, normalized-query."
  [request-context subscription]
  (let [native-id (:native-id subscription)
        provider-id (:provider-id subscription)
        metadata (-> (:metadata subscription) (json/decode true))
        normalized-query (:normalized-query subscription)
        collection-id (:CollectionConceptId metadata)
        subscriber-id (:SubscriberId metadata)
        ;; Find concepts with matching collection-concept-id, normalized-query, and subscriber-id
        duplicate-queries (mdb/find-concepts
                           request-context
                           {:collection-concept-id collection-id
                            :normalized-query normalized-query
                            :subscriber-id subscriber-id
                            :exclude-metadata true
                            :latest true}
                           :subscription)
        ;;we only want to look at non-deleted subscriptions
        active-duplicate-queries (remove :deleted duplicate-queries)]
     ;;If there is at least one duplicate subscription,
     ;;We need to make sure it has the same native-id, or else reject the ingest
     (when (and (> (count active-duplicate-queries) 0)
                (every? #(not= native-id (:native-id %)) active-duplicate-queries))
       (errors/throw-service-error
        :conflict
        (format (str "The subscriber-id [%s] has already subscribed to the "
                     "collection with concept-id [%s] using the query [%s]. "
                     "Subscribers must use unique queries for each Collection.")
                subscriber-id collection-id  normalized-query)))))

(defn- get-subscriber-id
  "Returns the subscriber id of the given subscription concept by parsing its metadata."
  [sub-concept]
  (-> sub-concept
      :metadata
      (json/decode true)
      :SubscriberId))

(defn- perform-subscription-ingest
  "This function assumes all checks have already taken place and that a
  subscription is ready to be saved"
  [request-context concept headers]
  (let [validated-concept (validate-and-prepare-subscription-concept concept)
        _ (check-subscriber-collection-permission request-context concept)
        concept-with-user-id (api-core/set-user-id validated-concept
                                                   request-context
                                                   headers)
        ;; Log the ingest attempt before the save
        _ (info (format "Ingesting subscription %s from client %s"
                        (api-core/concept->loggable-string concept-with-user-id)
                        (:client-id request-context)))
        save-subscription-result (ingest/save-subscription request-context
                                                           concept-with-user-id)]
    ;; Log the successful ingest, with the metadata size in bytes.
    (api-core/log-concept-with-metadata-size concept-with-user-id request-context)
    (api-core/generate-ingest-response headers save-subscription-result)))

(defn- common-ingest-checks
  "Common checks needed before starting to process an ingest operation"
  [request-context provider-id]
  (common-enabled/validate-write-enabled request-context "ingest")
  (lt-validation/validate-launchpad-token request-context)
  (api-core/verify-provider-exists request-context provider-id))

(defn- check-ingest-permission
  "Raise error if the user in the given context does not have the necessary permission to
   create/update/delete a subscription based on the new and old subscriber and existing ACLs."
  ([context provider-id new-subscriber]
   (check-ingest-permission context provider-id new-subscriber nil))
  ([context provider-id new-subscriber old-subscriber]
   (let [token-user (api-core/get-user-id-from-token context)]
     (if (and token-user
              (or (= token-user new-subscriber old-subscriber)
                  (and (= token-user new-subscriber)
                       (nil? old-subscriber))))
       (info (format "ACLs were bypassed because token user matches the subscriber [%s]."
                     new-subscriber))
       (do
         ;; log ACL checking message
         (if (and old-subscriber
                  (not= new-subscriber old-subscriber))
           (info (format "ACLs were checked because subscription subscriber is changed from [%s] to [%s]."
                         old-subscriber
                         new-subscriber))
           (info (format (str "ACLs were checked because token user [%s] is different from "
                              "the subscriber [%s] in the metadata.")
                         token-user
                         new-subscriber)))

         (acl/verify-ingest-management-permission
           context :update :provider-object provider-id)
         (acl/verify-subscription-management-permission
           context :update :provider-object provider-id))))))

(defn check-valid-user
  "Raise error if the user provided does not exist"
  [context user-id]
  (when (string/blank? user-id)
    (errors/throw-service-error
     :bad-request
     "Subscription creation failed - No ID was provided. Please provide a SubscriberId or pass in a valid token."))
  (when-not (urs/user-exists? context user-id) 
    (errors/throw-service-error
     :bad-request
     (format "Subscription creation failed - The user-id [%s] must correspond to a valid EDL account." 
             user-id))))

(defn generate-native-id
  "Generate a native-id for a subscription based on the name."
  [subscription]
  (-> subscription
      :metadata
      (json/parse-string true)
      :Name
      csk/->snake_case
      (str "_" (UUID/randomUUID))))

(defn native-id-collision?
  "Queries metadata db for a matching provider-id and native-id pair."
  [context provider-id native-id]
  (let [query {:provider-id provider-id
{'fraction_non_alphanumeric': 0.09790811933343463, 'fraction_numerical': 0.0005065086359722433, 'mean_word_length': 2.3664109121909633, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '406813', 'n_tokens_mistral': 5382, 'n_tokens_neox': 5225, 'n_words': 1574}