<commit_before>(ns cmr.common-app.services.kms-fetcher
  "Provides functions to easily fetch keywords from the GCMD Keyword Management Service (KMS). It
  will use a cache in order to minimize calls to the GCMD KMS and improve performance. The job
  defined in this namespace should be used to keep the KMS keywords fresh. KMS keywords will be
  cached using a fallback cache with Cubby as the backup store. See the documentation for
  cmr.common.cache.fallback-cache for more details. As a result of persisting the keywords in Cubby,
  the CMR will still be able to lookup KMS keywords even when the GCMD KMS is unavailable. CMR will
  use the last keyword values which were retrieved from the GCMD KMS before it became unavailable."
  (:require [cmr.common.services.errors :as errors]
            [cmr.common.time-keeper :as tk]
            [cmr.common.jobs :refer [def-stateful-job]]
            [cmr.transmit.kms :as kms]
            [cmr.common.log :as log :refer (debug info warn error)]
            [cmr.common.cache :as cache]
            [cmr.common.cache.fallback-cache :as fallback-cache]
            [cmr.common-app.cache.cubby-cache :as cubby-cache]
            [cmr.common-app.cache.consistent-cache :as consistent-cache]
            [clojure.set :as set]
            [cheshire.core :as json]))

(def keyword-schemes
  "The keyword schemes which we fetch from the GCMD KMS."
  [:providers :platforms :instruments])

(def kms-cache-key
  "The key used to store the KMS cache in the system cache map."
  :kms)

(defn create-kms-cache
  "Used to create the cache that will be used for caching KMS keywords. All applications caching
  KMS keywords should use the same fallback cache to ensure functionality even if GCMD KMS becomes
  unavailable."
  []
  (fallback-cache/create-fallback-cache
    (consistent-cache/create-consistent-cache)
    (cubby-cache/create-cubby-cache)))

(defn refresh-kms-cache
  "Refreshes the KMS keywords stored in the cache. This should be called from a background job on a
  timer to keep the cache fresh. This will throw an exception if there is a problem fetching the
  keywords from KMS. The caller is responsible for catching and logging the exception."
  [context]
  (let [cache (cache/context->cache context kms-cache-key)
        updated-keywords (into {}
                               (for [keyword-scheme keyword-schemes]
                                 [keyword-scheme (kms/get-keywords-for-keyword-scheme
                                                   context keyword-scheme)]))]
    (cache/set-value cache kms-cache-key updated-keywords)))

(defn get-full-hierarchy-for-short-name
  "Returns the full hierarchy for a given short name. If the provided short-name cannot be found,
  nil will be returned."
  [context keyword-scheme short-name]
  (let [cache (cache/context->cache context kms-cache-key)]
    (get-in (cache/get-value cache kms-cache-key) [keyword-scheme short-name])))
<commit_msg>CMR-1878: Update documentation and add lookup function for populating KMS cache<commit_after>(ns cmr.common-app.services.kms-fetcher
  "Provides functions to easily fetch keywords from the GCMD Keyword Management Service (KMS). It
  will use a cache in order to minimize calls to the GCMD KMS and improve performance. The job
  defined in this namespace should be used to keep the KMS keywords fresh. KMS keywords will be
  cached using a fallback cache with Cubby as the backup store. See the documentation for
  cmr.common.cache.fallback-cache for more details. As a result of persisting the keywords in Cubby,
  the CMR will still be able to lookup KMS keywords even when the GCMD KMS is unavailable. CMR will
  use the last keyword values which were retrieved from the GCMD KMS before it became unavailable.

  The KMS keywords are all cached under a single :kms key. The structure looks like the following:
  {:kms {:platforms [\"SN-1\" {:category \"C\" :series-entity \"S\"
                               :short-name \"SN-1\" :long-name \"LN\"}
                     \"SN-2\" {...}
                    ]}
         :providers [...]}"
  (:require [cmr.common.services.errors :as errors]
            [cmr.common.time-keeper :as tk]
            [cmr.common.jobs :refer [def-stateful-job]]
            [cmr.transmit.kms :as kms]
            [cmr.common.log :as log :refer (debug info warn error)]
            [cmr.common.cache :as cache]
            [cmr.common.cache.fallback-cache :as fallback-cache]
            [cmr.common-app.cache.cubby-cache :as cubby-cache]
            [cmr.common-app.cache.consistent-cache :as consistent-cache]
            [cmr.common.cache.single-thread-lookup-cache :as stl-cache]
            [clojure.set :as set]
            [cheshire.core :as json]))

(def keyword-schemes
  "The keyword schemes which we fetch from the GCMD KMS."
  [:providers :platforms :instruments])

(def kms-cache-key
  "The key used to store the KMS cache in the system cache map."
  :kms)

(defn create-kms-cache
  "Used to create the cache that will be used for caching KMS keywords. All applications caching
  KMS keywords should use the same fallback cache to ensure functionality even if GCMD KMS becomes
  unavailable."
  []
  (stl-cache/create-single-thread-lookup-cache
    (fallback-cache/create-fallback-cache
      (consistent-cache/create-consistent-cache)
      (cubby-cache/create-cubby-cache))))

(defn- get-all-gcmd-keywords-as-map
  "Calls GCMD KMS endpoints to retrieve the keywords. Response is a map structured in the same way
  as used in the KMS cache."
  [context]
  (into {}
        (for [keyword-scheme keyword-schemes]
          [keyword-scheme (kms/get-keywords-for-keyword-scheme
                            context keyword-scheme)])))

(defn refresh-kms-cache
  "Refreshes the KMS keywords stored in the cache. This should be called from a background job on a
  timer to keep the cache fresh. This will throw an exception if there is a problem fetching the
  keywords from KMS. The caller is responsible for catching and logging the exception."
  [context]
  (let [cache (cache/context->cache context kms-cache-key)]
    (cache/set-value cache kms-cache-key (get-all-gcmd-keywords-as-map context))))

(defn get-full-hierarchy-for-short-name
  "Returns the full hierarchy for a given short name. If the provided short-name cannot be found,
  nil will be returned."
  [context keyword-scheme short-name]
  (let [cache (cache/context->cache context kms-cache-key)]
    (get-in (cache/get-value cache kms-cache-key (partial get-all-gcmd-keywords-as-map context))
            [keyword-scheme short-name])))

{'fraction_non_alphanumeric': 0.0891990291262136, 'fraction_numerical': 0.001061893203883495, 'mean_word_length': 2.8555555555555556, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5709912', 'n_tokens_mistral': 1936, 'n_tokens_neox': 1900, 'n_words': 731}