<commit_before>(ns cmr.common.mime-types
  "Provides vars and functions for parsing and generating between MIME
  type and HTTP Content-Type strings and data formats supported by the
  CMR."
  (:refer-clojure :exclude [atom])
  (:require [clojure.string :as str]
            [cmr.common.util :as util]
            [cmr.common.services.errors :as svc-errors]
            [ring.middleware.format-response :as fr]))

;; (note: these maps will be populated as MIME types are declared)

(def base-mime-type-to-format
  "A map of MIME type strings to CMR data format keywords."
  {})

(def format->mime-type
  "A map of CMR data format keywords to MIME type strings."
  {})

;; Supported MIME types

(defmacro defmimetype
  "An elegant DSL for crafting beautiful mime-type definitions."
  [name string format-kw & format-aliases]
  (alter-var-root #'base-mime-type-to-format assoc string format-kw)
  (doseq [kw (conj format-aliases format-kw)]
    (alter-var-root #'format->mime-type assoc kw string))
  `(def ~name ~string))

(defmimetype json "application/json" :json)

(defmimetype xml "application/xml" :xml)

(defmimetype echo10 "application/echo10+xml" :echo10)

(defmimetype iso-smap "application/iso:smap+xml" :iso-smap :iso_smap)

(defmimetype iso "application/iso19115+xml" :iso19115 :iso)

(defmimetype dif "application/dif+xml" :dif)

(defmimetype dif10 "application/dif10+xml" :dif10)

(defmimetype csv "text/csv" :csv)

(defmimetype atom  "application/atom+xml" :atom)

(defmimetype kml "application/vnd.google-earth.kml+xml" :kml)

(defmimetype opendata "application/opendata+json" :opendata)

(defmimetype native "application/metadata+xml" :native)

(def any "*/*")

;; extra helpers

(def all-supported-mime-types
  "A superset of all mime types supported by any CMR applications."
  (keys base-mime-type-to-format))

(defn mime-type->format
  "Converts a mime-type into the format requested."
  ([mime-type]
   (mime-type->format mime-type json))
  ([mime-type default-mime-type]
   (if mime-type
     (or (get base-mime-type-to-format mime-type)
         (get base-mime-type-to-format default-mime-type))
     (get base-mime-type-to-format default-mime-type))))

;; Content-Type utilities

(defn with-charset
  "Returns a Content-Type header string with the given mime-type and charset."
  [mime-type charset]
  (str mime-type "; charset=" charset))

(defn with-utf-8
  "Returns mimetype with utf-8 charset specified."
  [mime-type]
  (with-charset mime-type "utf-8"))

(defn extract-mime-types
  "Extracts mime types from an accept header string according to RFC 2616 and returns them
  as combined type/sub-type strings in order of preference.
  Example from spec:

  audio/*; q=0.2, audio/basic

  \"SHOULD be interpreted as \"I prefer audio/basic, but send me any audio
   type if it is the best available after an 80% mark-down in quality.\"\"

  This function will return [\"audio/basic\" \"audio/*\"]

  Note that we do not currently handle asterisks and matching them. So \"*/xml\" would not match
  application/xml."
  [mime-type-str]
  (when mime-type-str
    (for [{:keys [sub-type type]} (fr/parse-accept-header* mime-type-str)]
      (str type "/" sub-type))))

(defn mime-type-from-headers
  "Returns first acceptable preferred mime-type based on the 'accept' or 'content-type' headers or the
  first preferred mime type if none of them were acceptable"
  ([headers]
   (mime-type-from-headers headers all-supported-mime-types))
  ([headers potential-mime-types]
   (let [headers (util/map-keys str/lower-case headers)]
     (some (set potential-mime-types)
           (concat (extract-mime-types (get headers "accept"))
                   (extract-mime-types (get headers "content-type")))))))

(defn path-w-extension->mime-type
  "Parses the search path with extension and returns the requested mime-type or nil if no extension
  was passed."
  [search-path-w-extension]
  (when-let [extension (second (re-matches #"[^.]+(?:\.(.+))$" search-path-w-extension))]
    (format->mime-type (keyword extension))))
<commit_msg>Refactoring mime type definitions to avoid build failures.<commit_after>(ns cmr.common.mime-types
  "Provides vars and functions for parsing and generating between MIME
  type and HTTP Content-Type strings and data formats supported by the
  CMR."
  (:refer-clojure :exclude [atom])
  (:require [clojure.string :as str]
            [cmr.common.util :as util]
            [cmr.common.services.errors :as svc-errors]
            [ring.middleware.format-response :as fr]))

(def mime-types
  "Defines a map of mime type format keywords to mime types and other format aliases. Each one of these
   gas a var defined for it for easy access."
  {:json {:mime-type "application/json"}
   :xml {:mime-type "application/xml"}
   :echo10 {:mime-type "application/echo10+xml"}
   :iso-smap {:mime-type "application/iso:smap+xml"
              :aliases [:iso_smap]}
   :iso19115 {:mime-type "application/iso19115+xml"
         :aliases [:iso]}
   :dif {:mime-type "application/dif+xml"}
   :dif10 {:mime-type "application/dif10+xml"}
   :csv {:mime-type "text/csv"}
   :atom {:mime-type  "application/atom+xml"}
   :kml {:mime-type "application/vnd.google-earth.kml+xml"}
   :opendata {:mime-type "application/opendata+json"}
   :native {:mime-type "application/metadata+xml"}})

;; Define vars for each of the mime type formats
(doseq [[format-kw {:keys [mime-type]}] mime-types]
  (eval `(def ~(symbol (name format-kw)) ~mime-type)))

(def iso
  "Defines a shorter alias for iso19115."
  iso19115)

(def any "*/*")

(def base-mime-type-to-format
  "A map of MIME type strings to CMR data format keywords."
  (into {} (for [[format-kw {:keys [mime-type]}] mime-types]
             [mime-type format-kw])))

(def format->mime-type
  "A map of CMR data format keywords to MIME type strings."
  (into {} (mapcat (fn [[format-kw {:keys [mime-type aliases]}]]
                     (cons [format-kw mime-type]
                            (for [a aliases]
                              [a mime-type])))
                   mime-types)))
;; extra helpers

(def all-supported-mime-types
  "A superset of all mime types supported by any CMR applications."
  (keys base-mime-type-to-format))

(defn mime-type->format
  "Converts a mime-type into the format requested."
  ([mime-type]
   (mime-type->format mime-type json))
  ([mime-type default-mime-type]
   (if mime-type
     (or (get base-mime-type-to-format mime-type)
         (get base-mime-type-to-format default-mime-type))
     (get base-mime-type-to-format default-mime-type))))

;; Content-Type utilities

(defn with-charset
  "Returns a Content-Type header string with the given mime-type and charset."
  [mime-type charset]
  (str mime-type "; charset=" charset))

(defn with-utf-8
  "Returns mimetype with utf-8 charset specified."
  [mime-type]
  (with-charset mime-type "utf-8"))

(defn extract-mime-types
  "Extracts mime types from an accept header string according to RFC 2616 and returns them
  as combined type/sub-type strings in order of preference.
  Example from spec:

  audio/*; q=0.2, audio/basic

  \"SHOULD be interpreted as \"I prefer audio/basic, but send me any audio
  type if it is the best available after an 80% mark-down in quality.\"\"

  This function will return [\"audio/basic\" \"audio/*\"]

  Note that we do not currently handle asterisks and matching them. So \"*/xml\" would not match
  application/xml."
  [mime-type-str]
  (when mime-type-str
    (for [{:keys [sub-type type]} (fr/parse-accept-header* mime-type-str)]
      (str type "/" sub-type))))

(defn mime-type-from-headers
  "Returns first acceptable preferred mime-type based on the 'accept' or 'content-type' headers or the
  first preferred mime type if none of them were acceptable"
  ([headers]
   (mime-type-from-headers headers all-supported-mime-types))
  ([headers potential-mime-types]
   (let [headers (util/map-keys str/lower-case headers)]
     (some (set potential-mime-types)
           (concat (extract-mime-types (get headers "accept"))
                   (extract-mime-types (get headers "content-type")))))))

(defn path-w-extension->mime-type
  "Parses the search path with extension and returns the requested mime-type or nil if no extension
  was passed."
  [search-path-w-extension]
  (when-let [extension (second (re-matches #"[^.]+(?:\.(.+))$" search-path-w-extension))]
    (format->mime-type (keyword extension))))

{'fraction_non_alphanumeric': 0.1337685459940653, 'fraction_numerical': 0.0085459940652819, 'mean_word_length': 3.4069037656903767, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 25}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2352774', 'n_tokens_mistral': 2922, 'n_tokens_neox': 2720, 'n_words': 854}