<commit_before>(ns cmr.system-int-test.bootstrap.bulk_index_test
  "Integration test for CMR bulk indexing."
  (:require [clojure.test :refer :all]
            [cmr.system-int-test.utils.ingest-util :as ingest]
            [cmr.umm.echo10.core :as echo10]
            [cmr.system-int-test.utils.search-util :as search]
            [cmr.system-int-test.utils.index-util :as index]
            [cmr.system-int-test.data2.collection :as dc]
            [cmr.system-int-test.data2.granule :as dg]
            [cmr.system-int-test.data2.core :as d]
            [cmr.common.config :as config]
            [clj-time.core :as t]))

(defn runnable-env?
  []
  (try
    (some-> 'user/system-type
            find-var
            var-get
            (= :external-dbs))
    (catch Exception e
      false)))

(use-fixtures :each (ingest/reset-fixture {"provguid1" "PROV1"}))
;; only run this test with the external db
(when (runnable-env?)
  ;; This test runs bulk index with some concepts in mdb that are good, and some that are
  ;; deleted, and some that have not yet been deleted, but have an expired deletion date.
  (deftest bulk-index-with-some-deleted
    (let [;; saved but not indexed
          coll1 {:short-name "coll1" :entry-title "coll1"}
          umm1 (dc/collection coll1)
          xml1 (echo10/umm->echo10-xml umm1)
          coll1-map {:concept-type :collection
                     :format "application/echo10+xml"
                     :metadata xml1
                     :extra-fields {:short-name "coll1"
                                    :entry-title "coll1"
                                    :version-id "v1"}
                     :provider-id "PROV1"
                     :native-id "coll1"
                     :short-name "coll1"}
          coll1 (ingest/save-concept coll1-map)
          umm1 (merge umm1 (select-keys coll1 [:concept-id :revision-id]))
          ;; saved and indexed by ingest
          coll2 {:short-name "coll2" :entry-title "coll2"}
          coll2 (d/ingest "PROV1" (dc/collection coll2))
          coll2-tombstone {:concept-id (:concept-id coll2)
                           :revision-id (inc (:revision-id coll2))}
          ;; saved (with old delete-time), but not indexed
          coll3 {:short-name "coll3" :entry-title "coll3" :delete-time "2000-01-01T12:00:00Z"}
          umm3 (dc/collection coll3)
          xml3 (echo10/umm->echo10-xml umm3)
          coll3-map {:concept-type :collection
                     :format "application/echo10+xml"
                     :metadata xml3
                     :extra-fields {:short-name "coll3"
                                    :entry-title "coll3"
                                    :version-id "v1"
                                    :delete-time "2000-01-01T12:00:00Z"}
                     :provider-id "PROV1"
                     :native-id "coll3"
                     :short-name "coll3"}
          coll3 (ingest/save-concept coll3-map)]
      (ingest/tombstone-concept coll2-tombstone)

      (index/bulk-index-provider "PROV1")
      (index/refresh-elastic-index)

      (testing "Expired documents are not indexed during bulk indexing and deleted documents
               get deleted."
               (are [search expected]
                    (d/refs-match? expected (search/find-refs :collection search))
                    {:concept-id (:concept-id coll1)} [umm1]
                    {:concept-id (:concept-id coll2)} []
                    {:concept-id (:concept-id coll3)} [])))))



;; This test verifies that the bulk indexer can run concurrently with ingest and indexing of items.
;; This test performs the following steps:
;; 1. Saves ten collections in metadata db.
;; 2. Saves three granules for each of those collections in metadata db.
;; 3. Ingests ten granules five times each in a separate thread.
;; 4. Concurrently executes a bulk index operation for the provider.
;; 5. Waits for the bulk indexing and granule ingest to complete.
;; 6. Searches for all of the saved/ingested concepts by concept-id.
;; 7. Verifies that the concepts returned by search have the expected revision ids.

;; only run this test with the external db
(when (runnable-env?)
  (deftest bulk-index-after-ingest
    (let [collections (for [x (range 1 11)]
                        (let [cmap {:short-name (str "short-name" x)
                                    :entry-title (str "ttl" x)}
                              umm (dc/collection cmap)
                              xml (echo10/umm->echo10-xml umm)
                              concept-map {:concept-type :collection
                                           :format "application/echo10+xml"
                                           :metadata xml
                                           :extra-fields {:short-name (str "short-name" x)
                                                          :entry-title (str "title" x)
                                                          :version-id "v1"}
                                           :provider-id "PROV1"
                                           :native-id (str "coll" x)
                                           :short-name (str "short-name" x)}]
                          (ingest/save-concept concept-map)))
          granules1 (mapcat (fn [collection]
                              (doall
                                (for [x (range 1 4)]
                                  (let [pid (:concept-id collection)
                                        cmap {:native-id (str "gran-" pid "-" x)}
                                        umm (dg/granule cmap)<commit_msg>Fixing the bulk-index-test so it works on reload.<commit_after>(ns cmr.system-int-test.bootstrap.bulk-index-test
  "Integration test for CMR bulk indexing."
  (:require [clojure.test :refer :all]
            [cmr.system-int-test.utils.ingest-util :as ingest]
            [cmr.umm.echo10.core :as echo10]
            [cmr.system-int-test.utils.search-util :as search]
            [cmr.system-int-test.utils.index-util :as index]
            [cmr.system-int-test.data2.collection :as dc]
            [cmr.system-int-test.data2.granule :as dg]
            [cmr.system-int-test.data2.core :as d]
            [cmr.common.config :as config]
            [clj-time.core :as t]))

(defn runnable-env?
  []
  (try
    (some-> 'user/system-type
            find-var
            var-get
            (= :external-dbs))
    (catch Exception e
      (.printStackTrace e)
      (println "Exception indicates this is not a runnable environment")
      false)))

(use-fixtures :each (ingest/reset-fixture {"provguid1" "PROV1"}))

;; This test runs bulk index with some concepts in mdb that are good, and some that are
;; deleted, and some that have not yet been deleted, but have an expired deletion date.
(deftest bulk-index-with-some-deleted
  ;; only run this test with the external db
  (when (runnable-env?)
    (let [;; saved but not indexed
          coll1 {:short-name "coll1" :entry-title "coll1"}
          umm1 (dc/collection coll1)
          xml1 (echo10/umm->echo10-xml umm1)
          coll1-map {:concept-type :collection
                     :format "application/echo10+xml"
                     :metadata xml1
                     :extra-fields {:short-name "coll1"
                                    :entry-title "coll1"
                                    :version-id "v1"}
                     :provider-id "PROV1"
                     :native-id "coll1"
                     :short-name "coll1"}
          coll1 (ingest/save-concept coll1-map)
          umm1 (merge umm1 (select-keys coll1 [:concept-id :revision-id]))
          ;; saved and indexed by ingest
          coll2 {:short-name "coll2" :entry-title "coll2"}
          coll2 (d/ingest "PROV1" (dc/collection coll2))
          coll2-tombstone {:concept-id (:concept-id coll2)
                           :revision-id (inc (:revision-id coll2))}
          ;; saved (with old delete-time), but not indexed
          coll3 {:short-name "coll3" :entry-title "coll3" :delete-time "2000-01-01T12:00:00Z"}
          umm3 (dc/collection coll3)
          xml3 (echo10/umm->echo10-xml umm3)
          coll3-map {:concept-type :collection
                     :format "application/echo10+xml"
                     :metadata xml3
                     :extra-fields {:short-name "coll3"
                                    :entry-title "coll3"
                                    :version-id "v1"
                                    :delete-time "2000-01-01T12:00:00Z"}
                     :provider-id "PROV1"
                     :native-id "coll3"
                     :short-name "coll3"}
          coll3 (ingest/save-concept coll3-map)]
      (ingest/tombstone-concept coll2-tombstone)

      (index/bulk-index-provider "PROV1")
      (index/refresh-elastic-index)

      (testing "Expired documents are not indexed during bulk indexing and deleted documents
               get deleted."
               (are [search expected]
                    (d/refs-match? expected (search/find-refs :collection search))
                    {:concept-id (:concept-id coll1)} [umm1]
                    {:concept-id (:concept-id coll2)} []
                    {:concept-id (:concept-id coll3)} [])))))



;; This test verifies that the bulk indexer can run concurrently with ingest and indexing of items.
;; This test performs the following steps:
;; 1. Saves ten collections in metadata db.
;; 2. Saves three granules for each of those collections in metadata db.
;; 3. Ingests ten granules five times each in a separate thread.
;; 4. Concurrently executes a bulk index operation for the provider.
;; 5. Waits for the bulk indexing and granule ingest to complete.
;; 6. Searches for all of the saved/ingested concepts by concept-id.
;; 7. Verifies that the concepts returned by search have the expected revision ids.

(deftest bulk-index-after-ingest
  ;; only run this test with the external db
  (when (runnable-env?)
    (let [collections (for [x (range 1 11)]
                        (let [cmap {:short-name (str "short-name" x)
                                    :entry-title (str "ttl" x)}
                              umm (dc/collection cmap)
                              xml (echo10/umm->echo10-xml umm)
                              concept-map {:concept-type :collection
                                           :format "application/echo10+xml"
                                           :metadata xml
                                           :extra-fields {:short-name (str "short-name" x)
                                                          :entry-title (str "title" x)
                                                          :version-id "v1"}
                                           :provider-id "PROV1"
                                           :native-id (str "coll" x)
                                           :short-name (str "short-name" x)}]
                          (ingest/save-concept concept-map)))
          granules1 (mapcat (fn [collection]
                              (doall
                                (for [x (range 1 4)]
                                  (let [pid (:concept-id collection)
                                        cmap {:native-id (str "gran-" pid "-" x)}
                                        umm (dg/granule cmap)
{'fraction_non_alphanumeric': 0.10468708623885603, 'fraction_numerical': 0.021537646747285727, 'mean_word_length': 1.4199060230670653, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 24}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 60, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '132773', 'n_tokens_mistral': 3284, 'n_tokens_neox': 3006, 'n_words': 887}