<commit_before>(ns cmr.system-int-test.utils.bootstrap-util
  "Contains utilities for working with the bootstrap application."
  (:require [cheshire.core :as json]
            [clojure.test :refer [is]]
            [clj-http.client :as client]
            [cmr.system-int-test.utils.url-helper :as url]
            [cmr.system-int-test.utils.ingest-util :as ingest]
            [cmr.system-int-test.utils.test-environment :as test-env]
            [cmr.metadata-db.config :as mdb-config]
            [cmr.bootstrap.test.catalog-rest :as cat-rest]
            [cmr.common.lifecycle :as lifecycle]
            [cmr.common.util :as util]
            [cmr.system-int-test.system :as s]
            [clj-time.core :as t]
            [clj-time.format :as f]))

(defn bulk-index-provider
  "Call the bootstrap app to bulk index a provider."
  [provider-id]
  (let [response (client/request
                   {:method :post
                    :query-params {:synchronous true}
                    :url (url/bulk-index-provider-url)
                    :body (format "{\"provider_id\": \"%s\"}" provider-id)
                    :content-type :json
                    :accept :json
                    :throw-exceptions false
                    :connection-manager (url/conn-mgr)})
        body (json/decode (:body response) true)]
    (assoc body :status (:status response))))

(defn bulk-index-collection
  "Call the bootstrap app to bulk index a collection."
  [provider-id collection-id]
  (let [response (client/request
                   {:method :post
                    :query-params {:synchronous true}
                    :url (url/bulk-index-collection-url)
                    :body (format "{\"provider_id\": \"%s\", \"collection_id\": \"%s\"}" provider-id collection-id)
                    :content-type :json
                    :accept :json
                    :throw-exceptions false
                    :connection-manager (url/conn-mgr)})
        body (json/decode (:body response) true)]
    (assoc body :status (:status response))))

(defn bulk-migrate-provider
  "Call the bootstrap app to bulk db migrate a provider."
  [provider-id]
  (let [response (client/request
                   {:method :post
                    :url (url/bulk-migrate-provider-url)
                    :query-params {:synchronous true}
                    :body (format "{\"provider_id\": \"%s\"}" provider-id)
                    :content-type :json
                    :accept :json
                    :throw-exceptions false
                    :connection-manager (url/conn-mgr)})
        body (json/decode (:body response) true)]
    (assoc body :status (:status response))))

(defn bulk-migrate-providers
  "Bulk migrates all the providers. Assumes they should all be successful"
  [& provider-ids]
  (doseq [provider-id provider-ids]
    (is (= 202
           (:status (bulk-migrate-provider provider-id))))))

(defn bulk-index-providers
  "Bulk indexes all the providers. Assumes they should all be successful"
  [& provider-ids]
  (doseq [provider-id provider-ids]
    (is (= 202
           (:status (bulk-index-provider provider-id))))))

(defn synchronize-databases
  "Tells the bootstrap application to synchronize Catalog REST and CMR."
  ([]
   (synchronize-databases {}))
  ([params]
   (let [{:keys [provider-id entry-title start-time end-time]} params
         start-time (when start-time (f/unparse (f/formatters :date-time) start-time))
         end-time (when end-time (f/unparse (f/formatters :date-time) end-time))
         query-params (util/remove-nil-keys
                        {:synchronous true
                         :start_date start-time
                         :end_date end-time
                         :provider_id provider-id
                         :entry_title entry-title})
         response (client/request {:method :post
                                   :url (url/db-synchronize-url)
                                   :query-params query-params
                                   :accept :json
                                   :throw-exceptions false
                                   :connection-manager (url/conn-mgr)})
         body (json/decode (:body response) true)]
     (assoc body :status (:status response)))))

(defn system
  "Returns a system suitable for calling the Catalog REST test code to create providers and add concepts."
  []
  {:db (:bootstrap-db (s/system))
   :catalog-rest-user (mdb-config/catalog-rest-db-username)})

(defn db-fixture-setup
  [& provider-ids]
  (test-env/only-with-real-database
    (let [system (system)]
      (ingest/reset)
      (doseq [provider-id provider-ids :let [guid (str provider-id "-guid")]]
        (ingest/create-provider guid provider-id true)
        (cat-rest/create-provider system provider-id)))))

(defn db-fixture-tear-down
  [& provider-ids]
  (test-env/only-with-real-database
    (let [system (system)]
      (ingest/reset)
      ;; Delete catalog rest providers
      (doseq [provider-id provider-ids]
        (cat-rest/delete-provider system provider-id)))))

(defn db-fixture
  "This is a fixture that sets up things for bootstrap database integration tests. It resets the CMR,
  then creates CMR providers and Catalog REST providers. All data is cleaned up at the end.
  If we're not connected to a real database then the setup is skipped."
  [& provider-ids]
  (fn [f]
    (try
      (apply db-fixture-setup provider-ids)
      (f)
      (finally
        (apply db-fixture-tear-down provider-ids)))))
<commit_msg>Changed string building using json/generate-string instead of 'format' fn in bootstrap_util.clj<commit_after>(ns cmr.system-int-test.utils.bootstrap-util
  "Contains utilities for working with the bootstrap application."
  (:require [cheshire.core :as json]
            [clojure.test :refer [is]]
            [clj-http.client :as client]
            [cmr.system-int-test.utils.url-helper :as url]
            [cmr.system-int-test.utils.ingest-util :as ingest]
            [cmr.system-int-test.utils.test-environment :as test-env]
            [cmr.metadata-db.config :as mdb-config]
            [cmr.bootstrap.test.catalog-rest :as cat-rest]
            [cmr.common.lifecycle :as lifecycle]
            [cmr.common.util :as util]
            [cmr.system-int-test.system :as s]
            [clj-time.core :as t]
            [clj-time.format :as f]))

(defn bulk-index-provider
  "Call the bootstrap app to bulk index a provider."
  [provider-id]
  (let [response (client/request
                   {:method :post
                    :query-params {:synchronous true}
                    :url (url/bulk-index-provider-url)
                    :body (json/generate-string {:provider_id provider-id})
                    :content-type :json
                    :accept :json
                    :throw-exceptions false
                    :connection-manager (url/conn-mgr)})
        body (json/decode (:body response) true)]
    (assoc body :status (:status response))))

(defn bulk-index-collection
  "Call the bootstrap app to bulk index a collection."
  [provider-id collection-id]
  (let [response (client/request
                   {:method :post
                    :query-params {:synchronous true}
                    :url (url/bulk-index-collection-url)
                    :body (json/generate-string {:provider_id provider-id :collection_id collection-id})
                    :content-type :json
                    :accept :json
                    :throw-exceptions false
                    :connection-manager (url/conn-mgr)})
        body (json/decode (:body response) true)]
    (assoc body :status (:status response))))

(defn bulk-migrate-provider
  "Call the bootstrap app to bulk db migrate a provider."
  [provider-id]
  (let [response (client/request
                   {:method :post
                    :url (url/bulk-migrate-provider-url)
                    :query-params {:synchronous true}
                    :body (format "{\"provider_id\": \"%s\"}" provider-id)
                    :content-type :json
                    :accept :json
                    :throw-exceptions false
                    :connection-manager (url/conn-mgr)})
        body (json/decode (:body response) true)]
    (assoc body :status (:status response))))

(defn bulk-migrate-providers
  "Bulk migrates all the providers. Assumes they should all be successful"
  [& provider-ids]
  (doseq [provider-id provider-ids]
    (is (= 202
           (:status (bulk-migrate-provider provider-id))))))

(defn bulk-index-providers
  "Bulk indexes all the providers. Assumes they should all be successful"
  [& provider-ids]
  (doseq [provider-id provider-ids]
    (is (= 202
           (:status (bulk-index-provider provider-id))))))

(defn synchronize-databases
  "Tells the bootstrap application to synchronize Catalog REST and CMR."
  ([]
   (synchronize-databases {}))
  ([params]
   (let [{:keys [provider-id entry-title start-time end-time]} params
         start-time (when start-time (f/unparse (f/formatters :date-time) start-time))
         end-time (when end-time (f/unparse (f/formatters :date-time) end-time))
         query-params (util/remove-nil-keys
                        {:synchronous true
                         :start_date start-time
                         :end_date end-time
                         :provider_id provider-id
                         :entry_title entry-title})
         response (client/request {:method :post
                                   :url (url/db-synchronize-url)
                                   :query-params query-params
                                   :accept :json
                                   :throw-exceptions false
                                   :connection-manager (url/conn-mgr)})
         body (json/decode (:body response) true)]
     (assoc body :status (:status response)))))

(defn system
  "Returns a system suitable for calling the Catalog REST test code to create providers and add concepts."
  []
  {:db (:bootstrap-db (s/system))
   :catalog-rest-user (mdb-config/catalog-rest-db-username)})

(defn db-fixture-setup
  [& provider-ids]
  (test-env/only-with-real-database
    (let [system (system)]
      (ingest/reset)
      (doseq [provider-id provider-ids :let [guid (str provider-id "-guid")]]
        (ingest/create-provider guid provider-id true)
        (cat-rest/create-provider system provider-id)))))

(defn db-fixture-tear-down
  [& provider-ids]
  (test-env/only-with-real-database
    (let [system (system)]
      (ingest/reset)
      ;; Delete catalog rest providers
      (doseq [provider-id provider-ids]
        (cat-rest/delete-provider system provider-id)))))

(defn db-fixture
  "This is a fixture that sets up things for bootstrap database integration tests. It resets the CMR,
  then creates CMR providers and Catalog REST providers. All data is cleaned up at the end.
  If we're not connected to a real database then the setup is skipped."
  [& provider-ids]
  (fn [f]
    (try
      (apply db-fixture-setup provider-ids)
      (f)
      (finally
        (apply db-fixture-tear-down provider-ids)))))

{'fraction_non_alphanumeric': 0.11784239813318974, 'fraction_numerical': 0.0010770059235325794, 'mean_word_length': 1.9502250463330686, 'pattern_counts': {'":': 5, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3227529', 'n_tokens_mistral': 3298, 'n_tokens_neox': 3142, 'n_words': 867}