<commit_before>(ns cmr.search.data.elastic-search-index
  "Implements the search index protocols for searching against Elasticsearch."
  (:require [clojure.string :as s]
            [clojurewerkz.elastisch.rest.document :as esd]
            [clojurewerkz.elastisch.query :as q]
            [cmr.common.log :refer (debug info warn error)]
            [cmr.common.util :as util]
            [cmr.common.lifecycle :as lifecycle]
            [cmr.common.jobs :refer [defjob]]
            [cmr.common.cache :as cache]
            [cmr.transmit.index-set :as index-set]
            [cmr.common-app.services.search.query-model :as qm]
            [cmr.common-app.services.search.query-to-elastic :as q2e]
            ;; Required to be available at runtime.
            [cmr.search.data.query-to-elastic]
            [cmr.search.services.query-walkers.collection-concept-id-extractor :as cex]
            [cmr.search.services.query-walkers.provider-id-extractor :as pex]
            [cmr.common.services.errors :as e]
            [cmr.common.concepts :as concepts]
            [cmr.common-app.services.search.elastic-search-index :as common-esi]))

;; id of the index-set that CMR is using, hard code for now
(def index-set-id 1)

(defn- fetch-concept-type-index-names
  "Fetch index names for each concept type from index-set app"
  [context]
  (let [fetched-index-set (index-set/get-index-set context index-set-id)
        rebalancing-collections (get-in fetched-index-set
                                        [:index-set :granule :rebalancing-collections])
        index-names-map (get-in fetched-index-set [:index-set :concepts])]
    ;; Remove any rebalancing collections from the set of granule index names we'll search through.
    {:index-names index-names-map
     :rebalancing-collections rebalancing-collections}))

(def index-cache-name
  "The name of the cache for caching index names. It will contain a map of concept type to a map of
   index names to the name of the index used in elasticsearch.

   Example:
   {:granule {:small_collections \"1_small_collections\"},
    :tag {:tags \"1_tags\"},
    :collection {:all-collection-revisions \"1_all_collection_revisions\",
                 :collections \"1_collections\"}}"
  :index-names)

;; A job for refreshing the index names cache.
(defjob RefreshIndexNamesCacheJob
  [ctx system]
  (let [context {:system system}
        index-names (fetch-concept-type-index-names context)
        cache (cache/context->cache context index-cache-name)]
    (cache/set-value cache index-cache-name index-names)))

(def refresh-index-names-cache-job
  {:job-type RefreshIndexNamesCacheJob
   ;; 5 minutes
   :interval 300})

(defn- get-index-names-map
  "Fetch index names associated with concepts."
  [context]
  (cache/get-value (cache/context->cache context index-cache-name)
                   :concept-indices
                   (partial fetch-concept-type-index-names context)))
(defn- get-granule-index-names
  "Fetch index names associated with granules excluding rebalancing collections indexes"
  [context]
  (let [{:keys [index-names rebalancing-collections]} (get-index-names-map context)]
    (apply dissoc (:granule index-names) (map keyword rebalancing-collections))))

(defn- collection-concept-id->index-name
  "Return the granule index name for the input collection concept id"
  [indexes coll-concept-id]
  (get indexes (keyword coll-concept-id) (get indexes :small_collections)))

(defn- collection-concept-ids->index-names
  "Return the granule index names for the input collection concept ids"
  [context coll-concept-ids]
  (let [indexes (get-granule-index-names context)]
    (distinct (map #(collection-concept-id->index-name indexes %) coll-concept-ids))))

(defn- provider-ids->index-names<commit_msg>CMR-2673: Fixing issues with the RefreshIndexNamesCacheJob<commit_after>(ns cmr.search.data.elastic-search-index
  "Implements the search index protocols for searching against Elasticsearch."
  (:require [clojure.string :as s]
            [clojurewerkz.elastisch.rest.document :as esd]
            [clojurewerkz.elastisch.query :as q]
            [cmr.common.log :refer (debug info warn error)]
            [cmr.common.util :as util]
            [cmr.common.lifecycle :as lifecycle]
            [cmr.common.jobs :refer [defjob]]
            [cmr.common.cache :as cache]
            [cmr.transmit.index-set :as index-set]
            [cmr.common-app.services.search.query-model :as qm]
            [cmr.common-app.services.search.query-to-elastic :as q2e]
            ;; Required to be available at runtime.
            [cmr.search.data.query-to-elastic]
            [cmr.search.services.query-walkers.collection-concept-id-extractor :as cex]
            [cmr.search.services.query-walkers.provider-id-extractor :as pex]
            [cmr.common.services.errors :as e]
            [cmr.common.concepts :as concepts]
            [cmr.common-app.services.search.elastic-search-index :as common-esi]))

;; id of the index-set that CMR is using, hard code for now
(def index-set-id 1)

(defn- fetch-concept-type-index-names
  "Fetch index names for each concept type from index-set app"
  [context]
  (let [fetched-index-set (index-set/get-index-set context index-set-id)
        rebalancing-collections (get-in fetched-index-set
                                        [:index-set :granule :rebalancing-collections])
        index-names-map (get-in fetched-index-set [:index-set :concepts])]
    ;; Remove any rebalancing collections from the set of granule index names we'll search through.
    {:index-names index-names-map
     :rebalancing-collections rebalancing-collections}))

(def index-cache-name
  "The name of the cache for caching index names. It will contain a map of concept type to a map of
   index names to the name of the index used in elasticsearch.

   Example:
   {:granule {:small_collections \"1_small_collections\"},
    :tag {:tags \"1_tags\"},
    :collection {:all-collection-revisions \"1_all_collection_revisions\",
                 :collections \"1_collections\"}}"
  :index-names)

(def index-names-cache-key
  "The key used for index names in the index cache."
  :concept-indices)

;; A job for refreshing the index names cache.
(defjob RefreshIndexNamesCacheJob
  [ctx system]
  (let [context {:system system}
        index-names (fetch-concept-type-index-names context)
        cache (cache/context->cache context index-cache-name)]
    (cache/set-value cache index-names-cache-key index-names)))

(def refresh-index-names-cache-job
  {:job-type RefreshIndexNamesCacheJob
   ;; 5 minutes
   :interval 300})

(defn- get-index-names-map
  "Fetch index names associated with concepts."
  [context]
  (cache/get-value (cache/context->cache context index-cache-name)
                   index-names-cache-key
                   (partial fetch-concept-type-index-names context)))

(defn- get-granule-index-names
  "Fetch index names associated with granules excluding rebalancing collections indexes"
  [context]
  (let [{:keys [index-names rebalancing-collections]} (get-index-names-map context)]
    (apply dissoc (:granule index-names) (map keyword rebalancing-collections))))

(defn- collection-concept-id->index-name
  "Return the granule index name for the input collection concept id"
  [indexes coll-concept-id]
  (get indexes (keyword coll-concept-id) (get indexes :small_collections)))

(defn- collection-concept-ids->index-names
  "Return the granule index names for the input collection concept ids"
  [context coll-concept-ids]
  (let [indexes (get-granule-index-names context)]
    (distinct (map #(collection-concept-id->index-name indexes %) coll-concept-ids))))

(defn- provider-ids->index-names
{'fraction_non_alphanumeric': 0.1154195305407859, 'fraction_numerical': 0.003112436778627934, 'mean_word_length': 3.0208550573514077, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7387996', 'n_tokens_mistral': 2437, 'n_tokens_neox': 2334, 'n_words': 627}