<commit_before>(ns cmr.system-int-test.search.granule-counts-search-test
  "This tests the granule counts search feature which allows retrieving counts of granules per collection."
  (:require [clojure.test :refer :all]
            [cmr.system-int-test.utils.ingest-util :as ingest]
            [cmr.system-int-test.utils.search-util :as search]
            [cmr.system-int-test.utils.index-util :as index]
            [cmr.system-int-test.data2.collection :as dc]
            [cmr.system-int-test.data2.granule :as dg]
            [cmr.system-int-test.data2.granule-counts :as gran-counts]
            [cmr.system-int-test.data2.core :as d]
            [cmr.spatial.codec :as codec]
            [cmr.spatial.point :as p]
            [cmr.common.util :as util]
            [cmr.spatial.mbr :as m]))


(use-fixtures :each (ingest/reset-fixture {"provguid1" "PROV1"}))

(comment

  (ingest/reset)
  (ingest/create-provider "provguid1" "PROV1")


  )

(defn temporal-range
  "Creates attributes for collection or granule defining a temporal range between start and stop
  which should be single digit integers."
  [start stop]
  (let [n-to-date-str #(str "200" % "-01-01T00:00:00Z")]
    {:beginning-date-time (n-to-date-str start)
     :ending-date-time (n-to-date-str stop)}))

(defn temporal-search-range
  "Creates a temporal search range between start and stop which should be single digit integers"
  [start stop]
  (let [{:keys [beginning-date-time ending-date-time]} (temporal-range start stop)]
    (str beginning-date-time "," ending-date-time)))

(deftest granule-related-collection-query-results-features-test
  (let [make-coll (fn [n shape temporal-attribs]
                    (let [spatial-attribs (when shape
                                            {:spatial-coverage
                                             (dc/spatial :geodetic :geodetic shape)})
                          coll-attribs (merge {:entry-title (str "coll" n)}
                                              spatial-attribs
                                              temporal-attribs)]
                      (d/ingest "PROV1" (dc/collection coll-attribs))))
        make-gran (fn [coll shape temporal-attribs]
                    (let [spatial-attribs (when shape {:spatial-coverage (dg/spatial shape)})
                          gran-attribs (merge {} spatial-attribs temporal-attribs)]
                      (d/ingest "PROV1" (dg/granule coll gran-attribs))))

        ;; Create collections
        ;; whole world, no temporal
        coll1 (make-coll 1 m/whole-world nil)
        ;; western hemisphere
        coll2 (make-coll 2 (m/mbr -180 90 0 -90) (temporal-range 1 3))
        ;; eastern hemisphere
        coll3 (make-coll 3 (m/mbr 0 90 180 -90) (temporal-range 2 4))
        ;; northern hemisphere
        coll4 (make-coll 4 (m/mbr -180 90 180 0) (temporal-range 3 5))
        ;; southern hemisphere
        coll5 (make-coll 5 (m/mbr -180 0 180 -90) (temporal-range 4 6))
        ;; no spatial
        coll6 (make-coll 6 nil (temporal-range 1 6))
        all-colls [coll1 coll2 coll3 coll4 coll5 coll6]]

    ;; -- Make granules --
    ;; Coll1 granules
    (make-gran coll1 (p/point 0 0) nil)
    (make-gran coll1 (p/point 0 90) nil)
    (make-gran coll1 (p/point 0 -90) nil)
    (make-gran coll1 (p/point -135 0) nil)
    (make-gran coll1 (p/point 135 0) nil)

    ;; Coll2 granules
    ;; There are none (should always be 0)

    ;; Coll3 granules
    (make-gran coll3 (p/point 0 90) (temporal-range 2 2))
    (make-gran coll3 (p/point 135 0) (temporal-range 3 3))
    (make-gran coll3 (p/point 0 -90) (temporal-range 4 4))

    ;; Coll4 granules
    (make-gran coll4 (p/point 0 90) (temporal-range 3 3))
    (make-gran coll4 (p/point 135 45) (temporal-range 4 4))
    (make-gran coll4 (p/point -135 45) (temporal-range 5 5))

    ;; Coll5 granules
    (make-gran coll5 (p/point 0 -90) (temporal-range 4 4))
    (make-gran coll5 (p/point 135 -45) (temporal-range 5 5))
    (make-gran coll5 (p/point -135 -45) (temporal-range 6 6))

    ;; Coll6 granules
    (make-gran coll6 nil (temporal-range 1 1))
    (make-gran coll6 nil (temporal-range 3 3))
    (make-gran coll6 nil (temporal-range 6 6))
    (index/refresh-elastic-index)

    (testing "granule counts"
      (testing "invalid include-granule-counts"
        (is (= {:errors ["Parameter include_granule_counts must take value of true, false, or unset, but was foo"] :status 400}
               (search/find-refs :collection {:include-granule-counts "foo"})))
        (is (= {:errors ["Parameter [include_granule_counts] was not recognized."] :status 400}
               (search/find-refs :granule {:include-granule-counts true}))))

      (testing "granule counts for all collections"
        (let [refs (search/find-refs :collection {:include-granule-counts true})]
          (is (gran-counts/granule-counts-match? :xml {coll1 5 coll2 0 coll3 3 coll4 3 coll5 3 coll6 3} refs))))

      (testing "granule counts for spatial queries"
        (are [wnes expected-counts]
             (let [refs (search/find-refs :collection {:include-granule-counts true
                                                       :bounding-box (codec/url-encode (apply m/mbr wnes))})]
               (gran-counts/granule-counts-match? :xml expected-counts refs))

             ;; Whole world
             [-180 90 180 -90] {coll1 5 coll2 0 coll3 3 coll4 3 coll5 3}<commit_msg>Added search by science keywords test with granule counts.<commit_after>(ns cmr.system-int-test.search.granule-counts-search-test
  "This tests the granule counts search feature which allows retrieving counts of granules per collection."
  (:require [clojure.test :refer :all]
            [cmr.system-int-test.utils.ingest-util :as ingest]
            [cmr.system-int-test.utils.search-util :as search]
            [cmr.system-int-test.utils.index-util :as index]
            [cmr.system-int-test.data2.collection :as dc]
            [cmr.system-int-test.data2.granule :as dg]
            [cmr.system-int-test.data2.granule-counts :as gran-counts]
            [cmr.system-int-test.data2.core :as d]
            [cmr.spatial.codec :as codec]
            [cmr.spatial.point :as p]
            [cmr.common.util :as util]
            [cmr.spatial.mbr :as m]))


(use-fixtures :each (ingest/reset-fixture {"provguid1" "PROV1"}))

(comment

  (ingest/reset)
  (ingest/create-provider "provguid1" "PROV1")


  )

(defn temporal-range
  "Creates attributes for collection or granule defining a temporal range between start and stop
  which should be single digit integers."
  [start stop]
  (let [n-to-date-str #(str "200" % "-01-01T00:00:00Z")]
    {:beginning-date-time (n-to-date-str start)
     :ending-date-time (n-to-date-str stop)}))

(defn temporal-search-range
  "Creates a temporal search range between start and stop which should be single digit integers"
  [start stop]
  (let [{:keys [beginning-date-time ending-date-time]} (temporal-range start stop)]
    (str beginning-date-time "," ending-date-time)))

(defn make-coll
  ([n shape temporal-attribs]
   (make-coll n shape temporal-attribs {}))
  ([n shape temporal-attribs other-attribs]
   (let [spatial-attribs (when shape
                           {:spatial-coverage
                            (dc/spatial :geodetic :geodetic shape)})
         coll-attribs (merge {:entry-title (str "coll" n)}
                             spatial-attribs
                             temporal-attribs
                             other-attribs)]
     (d/ingest "PROV1" (dc/collection coll-attribs)))))

(defn make-gran
  [coll shape temporal-attribs]
  (let [spatial-attribs (when shape {:spatial-coverage (dg/spatial shape)})
        gran-attribs (merge {} spatial-attribs temporal-attribs)]
    (d/ingest "PROV1" (dg/granule coll gran-attribs))))

(deftest granule-related-collection-query-results-features-test
  (let [;; Create collections
        ;; whole world, no temporal, and science keywords
        coll1 (make-coll 1 m/whole-world nil {:science-keywords
                                              [(dc/science-keyword {:category "Tornado"
                                                                    :topic "Wind"
                                                                    :term "Speed"})]})
        ;; western hemisphere
        coll2 (make-coll 2 (m/mbr -180 90 0 -90) (temporal-range 1 3))
        ;; eastern hemisphere
        coll3 (make-coll 3 (m/mbr 0 90 180 -90) (temporal-range 2 4))
        ;; northern hemisphere
        coll4 (make-coll 4 (m/mbr -180 90 180 0) (temporal-range 3 5))
        ;; southern hemisphere
        coll5 (make-coll 5 (m/mbr -180 0 180 -90) (temporal-range 4 6))
        ;; no spatial
        coll6 (make-coll 6 nil (temporal-range 1 6))
        all-colls [coll1 coll2 coll3 coll4 coll5 coll6]]

    ;; -- Make granules --
    ;; Coll1 granules
    (make-gran coll1 (p/point 0 0) nil)
    (make-gran coll1 (p/point 0 90) nil)
    (make-gran coll1 (p/point 0 -90) nil)
    (make-gran coll1 (p/point -135 0) nil)
    (make-gran coll1 (p/point 135 0) nil)

    ;; Coll2 granules
    ;; There are none (should always be 0)

    ;; Coll3 granules
    (make-gran coll3 (p/point 0 90) (temporal-range 2 2))
    (make-gran coll3 (p/point 135 0) (temporal-range 3 3))
    (make-gran coll3 (p/point 0 -90) (temporal-range 4 4))

    ;; Coll4 granules
    (make-gran coll4 (p/point 0 90) (temporal-range 3 3))
    (make-gran coll4 (p/point 135 45) (temporal-range 4 4))
    (make-gran coll4 (p/point -135 45) (temporal-range 5 5))

    ;; Coll5 granules
    (make-gran coll5 (p/point 0 -90) (temporal-range 4 4))
    (make-gran coll5 (p/point 135 -45) (temporal-range 5 5))
    (make-gran coll5 (p/point -135 -45) (temporal-range 6 6))

    ;; Coll6 granules
    (make-gran coll6 nil (temporal-range 1 1))
    (make-gran coll6 nil (temporal-range 3 3))
    (make-gran coll6 nil (temporal-range 6 6))
    (index/refresh-elastic-index)

    (testing "granule counts"
      (testing "invalid include-granule-counts"
        (is (= {:errors ["Parameter include_granule_counts must take value of true, false, or unset, but was foo"] :status 400}
               (search/find-refs :collection {:include-granule-counts "foo"})))
        (is (= {:errors ["Parameter [include_granule_counts] was not recognized."] :status 400}
               (search/find-refs :granule {:include-granule-counts true}))))

      (testing "granule counts for all collections"
        (let [refs (search/find-refs :collection {:include-granule-counts true})]
          (is (gran-counts/granule-counts-match? :xml {coll1 5 coll2 0 coll3 3 coll4 3 coll5 3 coll6 3} refs))))

      ;; CMR-712
      (testing "granule counts with science keywords query"
        (let [refs (search/find-refs :collection {:include-granule-counts true
                                                  :science-keywords {:0 {:category "Tornado"}}})]
          (is (gran-counts/granule-counts-match? :xml {coll1 5} refs))))

      (testing "granule counts for spatial queries"
        (are [wnes expected-counts]
             (let [refs (search/find-refs :collection {:include-granule-counts true
                                                       :bounding-box (codec/url-encode (apply m/mbr wnes))})]
               (gran-counts/granule-counts-match? :xml expected-counts refs))

             ;; Whole world
             [-180 90 180 -90] {coll1 5 coll2 0 coll3 3 coll4 3 coll5 3}
{'fraction_non_alphanumeric': 0.12503276539973787, 'fraction_numerical': 0.03949322848405417, 'mean_word_length': 2.1856387419983303, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 5}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2403207', 'n_tokens_mistral': 4192, 'n_tokens_neox': 3566, 'n_words': 1065}