<commit_before>(ns cmr.search.api.search-results
  "Contains functions for validating search results requested formats and for converting to
  requested format"
  (:require [cheshire.core :as json]
            [pantomime.media :as mt]
            [cmr.common.services.errors :as errors]
            [cmr.common.concepts :as ct]
            [clojure.data.xml :as x]
            [clojure.set :as set]
            [clojure.data.csv :as csv]
            [cmr.search.models.results :as r]
            [cmr.transmit.transformer :as t])
  (:import
    [java.io StringWriter]))


(def default-search-result-format :json)

(def base-mime-type-to-format
  "A map of base mime types to the format symbols supported"
  {"*/*" default-search-result-format
   "application/json" :json
   "application/xml" :xml
   "text/csv" :csv
   "application/echo10+xml" :echo10})

(defn mime-type->format
  "Converts a mime-type into the format requested."
  [mime-type]
  (if mime-type
    (get base-mime-type-to-format
         (str (mt/base-type (mt/parse mime-type))))
    default-search-result-format))

(def format->mime-type
  {:json "application/json"
   :xml "application/xml"
   :csv "text/csv"
   :echo10 "application/echo10+xml"})

(defn validate-search-result-mime-type
  "Validates the requested search result mime type."
  [mime-type]
  (when-not (mime-type->format mime-type)
    (errors/throw-service-error
      :bad-request (format "The mime type [%s] is not supported for search results." mime-type))))

(defmulti search-results->response
  (fn [context results result-type pretty]
    result-type))

(defmethod search-results->response :json
  [context results result-type pretty]
  (let [{:keys [hits took references]} results
        response-refs (map #(set/rename-keys % {:concept-id :id}) references)
        response-results (r/->Results hits took response-refs)]
    (json/generate-string response-results {:pretty pretty})))

(defn- reference->xml-element
  "Converts a search result reference into an XML element"
  [reference]
  (let [{:keys [concept-id revision-id location name]} reference]
    (x/element :reference {}
               (x/element :name {} name)
               (x/element :id {} concept-id)
               (x/element :location {} location)
               (x/element :revision-id {} (str revision-id)))))

(defmulti reference+echo10->xml-element
  "Converts a search result + echo10 data into an XML element"
  (fn [reference echo10-xml]
    (ct/concept-id->type (:concept-id reference))))

(defmethod reference+echo10->xml-element :granule
  [reference echo10-xml]
  (let [{:keys [concept-id parent-collection-id]} reference]
    (x/element :result
               {:echo_granule_id concept-id :echo_dataset_id parent-collection-id}
               echo10-xml)))

(defmethod reference+echo10->xml-element :collection
  [reference echo10-xml]
  (let [{:keys [concept-id]} reference]
    (x/element :result
               {:echo_dataset_id concept-id}
               echo10-xml)))

(defn- references->format
  "Converts search result references into the desired format"
  [context references format]
  (let [tuples (map #(vector (:concept-id %) (:revision-id %)) references)]
    (t/get-formatted-concept-revisions context tuples format)))

(defmethod search-results->response :xml
  [context results result-typ pretty]
  (let [{:keys [hits took references]} results
        xml-fn (if pretty x/indent-str x/emit-str)]
    (xml-fn
      (x/element :results {}
                 (x/element :hits {} (str hits))
                 (x/element :took {} (str took))
                 (x/->Element :references {}
                              (map reference->xml-element references))))))

(def CSV_HEADER
  ["Granule UR","Producer Granule ID","Start Time","End Time","Online Access URLs","Browse URLs","Cloud Cover","Day/Night","Size"])

(defmethod search-results->response :csv
  [context results result-type pretty]
  (let [{:keys [hits took references]} results
        response-refs (conj references CSV_HEADER)
        string-writer (StringWriter.)]
    (csv/write-csv string-writer response-refs)
    (str string-writer)))

(defmethod search-results->response :echo10
  [context results result-type pretty]
  (let [{:keys [hits took references]} results
        echo10 (references->format context references :echo10)
        xml-fn (if pretty x/indent-str x/emit-str)]
    (xml-fn
      (x/element :results {}
                 (x/element :hits {} (str hits))
                 (x/element :took {} (str took))
                 (map (fn [reference echo10-xml]
                        (reference+echo10->xml-element reference echo10-xml))
                      references echo10)))))
<commit_msg>CRM-564: Added code to strip out xml processing instructions form embedded echo10<commit_after>(ns cmr.search.api.search-results
  "Contains functions for validating search results requested formats and for converting to
  requested format"
  (:require [cheshire.core :as json]
            [pantomime.media :as mt]
            [cmr.common.services.errors :as errors]
            [cmr.common.concepts :as ct]
            [clojure.data.xml :as x]
            [clojure.set :as set]
            [clojure.data.csv :as csv]
            [clojure.string :as s]
            [cmr.search.models.results :as r]
            [cmr.transmit.transformer :as t])
  (:import
    [java.io StringWriter]))


(def default-search-result-format :json)

(def base-mime-type-to-format
  "A map of base mime types to the format symbols supported"
  {"*/*" default-search-result-format
   "application/json" :json
   "application/xml" :xml
   "text/csv" :csv
   "application/echo10+xml" :echo10})

(defn mime-type->format
  "Converts a mime-type into the format requested."
  [mime-type]
  (if mime-type
    (get base-mime-type-to-format
         (str (mt/base-type (mt/parse mime-type))))
    default-search-result-format))

(def format->mime-type
  {:json "application/json"
   :xml "application/xml"
   :csv "text/csv"
   :echo10 "application/echo10+xml"})

(defn validate-search-result-mime-type
  "Validates the requested search result mime type."
  [mime-type]
  (when-not (mime-type->format mime-type)
    (errors/throw-service-error
      :bad-request (format "The mime type [%s] is not supported for search results." mime-type))))

(defmulti search-results->response
  (fn [context results result-type pretty]
    result-type))

(defmethod search-results->response :json
  [context results result-type pretty]
  (let [{:keys [hits took references]} results
        response-refs (map #(set/rename-keys % {:concept-id :id}) references)
        response-results (r/->Results hits took response-refs)]
    (json/generate-string response-results {:pretty pretty})))

(defn- reference->xml-element
  "Converts a search result reference into an XML element"
  [reference]
  (let [{:keys [concept-id revision-id location name]} reference]
    (x/element :reference {}
               (x/element :name {} name)
               (x/element :id {} concept-id)
               (x/element :location {} location)
               (x/element :revision-id {} (str revision-id)))))

(defn- remove-xml-processing-instructions
  "Removes xml processing instructions from XML so it can be embedded in another XML document"
  [xml]
  (let [processing-regex #"<\?.*?\?>"
        doctype-regex #"<!DOCTYPE.*?>"]
    (-> xml
        (s/replace processing-regex "")
        (s/replace doctype-regex ""))))

(defmulti reference+echo10->xml-element
  "Converts a search result + echo10 data into an XML element"
  (fn [reference echo10-xml]
    (ct/concept-id->type (:concept-id reference))))

(defmethod reference+echo10->xml-element :granule
  [reference echo10-xml]
  (let [{:keys [concept-id parent-collection-id]} reference]
    (x/element :result
               {:echo_granule_id concept-id :echo_dataset_id parent-collection-id}
               echo10-xml)))

(defmethod reference+echo10->xml-element :collection
  [reference echo10-xml]
  (let [{:keys [concept-id]} reference]
    (x/element :result
               {:echo_dataset_id concept-id}
               echo10-xml)))

(defn- references->format
  "Converts search result references into the desired format"
  [context references format]
  (let [tuples (map #(vector (:concept-id %) (:revision-id %)) references)]
    (t/get-formatted-concept-revisions context tuples format)))

(defmethod search-results->response :xml
  [context results result-typ pretty]
  (let [{:keys [hits took references]} results
        xml-fn (if pretty x/indent-str x/emit-str)]
    (xml-fn
      (x/element :results {}
                 (x/element :hits {} (str hits))
                 (x/element :took {} (str took))
                 (x/->Element :references {}
                              (map reference->xml-element references))))))

(def CSV_HEADER
  ["Granule UR","Producer Granule ID","Start Time","End Time","Online Access URLs","Browse URLs","Cloud Cover","Day/Night","Size"])

(defmethod search-results->response :csv
  [context results result-type pretty]
  (let [{:keys [hits took references]} results
        response-refs (conj references CSV_HEADER)
        string-writer (StringWriter.)]
    (csv/write-csv string-writer response-refs)
    (str string-writer)))

(defmethod search-results->response :echo10
  [context results result-type pretty]
  (let [{:keys [hits took references]} results
        echo10 (references->format context references :echo10)
        xml-fn (if pretty x/indent-str x/emit-str)]
    (xml-fn
      (x/element :results {}
                 (x/element :hits {} (str hits))
                 (x/element :took {} (str took))
                 (map (fn [reference echo10-xml]
                        (reference+echo10->xml-element reference
                                                       (remove-xml-processing-instructions echo10-xml)))
                      references echo10)))))

{'fraction_non_alphanumeric': 0.13383049481003728, 'fraction_numerical': 0.008565957875642446, 'mean_word_length': 2.771949828962372, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 56}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5434590', 'n_tokens_mistral': 3099, 'n_tokens_neox': 2954, 'n_words': 810}