<commit_before>(ns cmr.search.api.keyword
  "Defines the HTTP URL routes for the keyword endpoint in the search application."
  (:require [compojure.core :refer :all]
            [cheshire.core :as json]
            [camel-snake-kebab.core :as csk]
            [cmr.common-app.services.kms-fetcher :as kf]
            [cmr.transmit.kms :as kms]
            [cmr.common.mime-types :as mt]
            [cmr.common.services.errors :as errors]
            [cmr.search.services.query-execution.facets-results-feature :as frf]))

(defn- validate-keyword-scheme
  "Throws a service error if the provided keyword-scheme is invalid."
  [keyword-scheme]
  (when-not (some? (keyword-scheme kms/keyword-scheme->field-names))
    (errors/throw-service-error
      :bad-request (format "The keyword scheme [%s] is not supported. Valid schemes are: %s"
                           (name keyword-scheme)
                           (pr-str (map name (keys kms/keyword-scheme->field-names)))))))

; (defn- parse-hierarchical-keywords
;   "Returns keywords in a hierarchical fashion for the given keyword scheme and keywords."
;   [keyword-scheme keyword-hierarchy keywords]
;   (when-let [subfield (first keyword-hierarchy)]
;     (let [all-values-for-subfield (map subfield keywords)]
;       (for [value all-values-for-subfield]
;         (parse-hierarchical-keywords keyword-scheme (rest keyword-hierarchy) keywords)))))

(def keyword-api-routes
  (context "/keywords" []
    ;; Return a list of keywords for the given scheme
    (GET "/:keyword-scheme" {{:keys [keyword-scheme] :as params} :params
                             request-context :request-context}
      (let [keyword-scheme (csk/->kebab-case-keyword keyword-scheme)]
        (validate-keyword-scheme keyword-scheme)
        (let [keywords (vals (keyword-scheme (kf/get-gcmd-keywords-map request-context)))]
          {:staus 200
           :headers {"Content-Type" (mt/format->mime-type :json)}
           :body (json/generate-string keywords)})))))


<commit_msg>CMR-1980: Start testing returning keywords hierarchically from keyword endpoint<commit_after>(ns cmr.search.api.keyword
  "Defines the HTTP URL routes for the keyword endpoint in the search application."
  (:require [compojure.core :refer :all]
            [cheshire.core :as json]
            [camel-snake-kebab.core :as csk]
            [cmr.common-app.services.kms-fetcher :as kf]
            [cmr.transmit.kms :as kms]
            [cmr.common.mime-types :as mt]
            [cmr.common.util :as util]
            [cmr.common.services.errors :as errors]
            [cmr.search.services.query-execution.facets-results-feature :as frf]))

(defn- validate-keyword-scheme
  "Throws a service error if the provided keyword-scheme is invalid."
  [keyword-scheme]
  (when-not (some? (keyword-scheme kms/keyword-scheme->field-names))
    (errors/throw-service-error
      :bad-request (format "The keyword scheme [%s] is not supported. Valid schemes are: %s"
                           (name keyword-scheme)
                           (pr-str (map name (keys kms/keyword-scheme->field-names)))))))

; (defn- parse-hierarchical-keywords
;   "Returns keywords in a hierarchical fashion for the given keyword scheme and keywords."
;   [keyword-scheme keyword-hierarchy keywords]
;   (when-let [subfield (first keyword-hierarchy)]
;     (let [all-values-for-subfield (map subfield keywords)]
;       (for [value all-values-for-subfield]
;         (parse-hierarchical-keywords keyword-scheme (rest keyword-hierarchy) keywords)))))


;; Sample output

(comment
  ["0dd83b2a-e83f-4a0c-a1ff-2fbdbbcce62d"
   {:uuid "0dd83b2a-e83f-4a0c-a1ff-2fbdbbcce62d",
    :variable-level-1 "MOBILE GEOGRAPHIC INFORMATION SYSTEMS",
    :term "GEOGRAPHIC INFORMATION SYSTEMS",
    :topic "DATA ANALYSIS AND VISUALIZATION",
    :category "EARTH SCIENCE SERVICES"}]

  {:category [{:value "EARTH SCIENCE SERVICES"
               :uuid "894f9116-ae3c-40b6-981d-5113de961710"
               :subfields ["topic"]
               :topic [{:value "DATA ANALYSIS AND VISUALIZATION"
                        :uuid "41adc080-c182-4753-9666-435f8b1c913f"
                        :subfields ["term"]
                        :term [{:value "GEOGRAPHIC INFORMATION SYSTEMS"
                                :uuid "794e3c3b-791f-44de-9ff3-358d8ed74733"
                                :subfields ["variable-level-1"]
                                :variable-level-1 [{:value "MOBILE GEOGRAPHIC INFORMATION SYSTEMS"
                                                    :uuid "0dd83b2a-e83f-4a0c-a1ff-2fbdbbcce62d"
                                                    }]}]}]}]}

  (def keyword-hierarchy
    [:category :topic :term :variable-level-1 :variable-level-2 :variable-level-3])

  (def keywords
    [{:uuid "0dd83b2a-e83f-4a0c-a1ff-2fbdbbcce62d",
      :variable-level-1 "MOBILE GEOGRAPHIC INFORMATION SYSTEMS",
      :term "GEOGRAPHIC INFORMATION SYSTEMS",
      :topic "DATA ANALYSIS AND VISUALIZATION",
      :category "EARTH SCIENCE SERVICES"}
     {:uuid "894f9116-ae3c-40b6-981d-5113de961710",
      :category "EARTH SCIENCE SERVICES"}
     {:uuid "41adc080-c182-4753-9666-435f8b1c913f",
      :topic "DATA ANALYSIS AND VISUALIZATION",
      :category "EARTH SCIENCE SERVICES"}
     {:uuid "e9f67a66-e9fc-435c-b720-ae32a2c3d8f5",
      :category "EARTH SCIENCE"}])

  {:category (map (fn [k-word] (util/remove-nil-keys {:value (:category k-word)
                                                      :uuid (when-not (:topic k-word)
                                                              (:uuid k-word))})) keywords)}

  (let [all-categories (keep (fn [k-word] (when-not (:topic k-word)
                                            {:value (:category k-word)
                                             :uuid  (:uuid k-word)})) keywords)
        all-topics (for [category all-categories
                         :let [topics (keep (fn [k-word]
                                              (when (and (nil? (:term k-word))
                                                         (:topic k-word)
                                                         (= (:value category) (:category k-word)))
                                                {:value (:topic k-word)
                                                 :uuid (:uuid k-word)})) keywords)]]
                     (if (seq topics)
                       (assoc category
                              :subfields ["topic"]
                              :topic topics)
                       category))]
    {:category all-topics})


  (parse-hierarchical-keywords :science-keywords keyword-hierarchy keywords)
 )

(def keyword-api-routes
  (context "/keywords" []
    ;; Return a list of keywords for the given scheme
    (GET "/:keyword-scheme" {{:keys [keyword-scheme] :as params} :params
                             request-context :request-context}
      (let [keyword-scheme (csk/->kebab-case-keyword keyword-scheme)]
        (validate-keyword-scheme keyword-scheme)
        (let [keywords (vals (keyword-scheme (kf/get-gcmd-keywords-map request-context)))]
          {:staus 200
           :headers {"Content-Type" (mt/format->mime-type :json)}
           :body (json/generate-string keywords)})))))



{'fraction_non_alphanumeric': 0.12146697918102854, 'fraction_numerical': 0.02757479663587481, 'mean_word_length': 1.9668711656441717, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6948178', 'n_tokens_mistral': 2340, 'n_tokens_neox': 2152, 'n_words': 508}