<commit_before>(ns cmr.indexer.data.concepts.attribute
  "Contains functions for converting attributes into a elastic documents"
  (:require [clj-time.format :as f]
            [clojure.string :as s]
            [cmr.umm.collection.product-specific-attribute :as coll-psa]
            [cmr.common.services.errors :as errors]))

(defmulti value->elastic-value
  "Converts a attribute value into the elastic value that should be transmitted"
  (fn [type value]
    type))

(defmethod value->elastic-value :default
  [type value]
  value)

(defmethod value->elastic-value :datetime
  [type value]
  ;; clj-time treats nil values as *now*. Adding explicit checks here to avoid weird errors.
  (when-not value (errors/internal-error! "Value is null"))
  (f/unparse (f/formatters :date-time) value))

(defmethod value->elastic-value :time
  [type value]
  ;; clj-time treats nil values as *now*. Adding explicit checks here to avoid weird errors.
  (when-not value (errors/internal-error! "Value is null"))
  ;; This relies on the fact that times are parsed into times on day 1970-01-01
  (f/unparse (f/formatters :date-time) value))

(defmethod value->elastic-value :date
  [type value]
  ;; clj-time treats nil values as *now*. Adding explicit checks here to avoid weird errors.
  (when-not value (errors/internal-error! "Value is null"))
  (f/unparse (f/formatters :date-time) value))

(def type->field-name
  "Converts an attribute type into the indexed field name"
  {:string "string-value"
   :time "time-value"
   :date "date-value"
   :datetime "datetime-value"
   :boolean "string-value"
   :time-string "string-value"
   :date-string "string-value"
   :datetime-string "string-value"
   :int "int-value"
   :float "float-value"})

(defn psa-ref->elastic-doc
  "Converts a PSA ref into the portion going in an elastic document"
  [type psa-ref]
  (let [field-name (type->field-name type)]
    (if (some #{type} [:string :boolean :time-string :date-string :datetime-string])
      [{:name (:name psa-ref)
        field-name (map (comp #(value->elastic-value type %)
                              #(coll-psa/parse-value type %))
                        (:values psa-ref))}
       {:name (:name psa-ref)
        (str field-name ".lowercase") (map (comp s/lower-case
                                                 #(value->elastic-value type %)
                                                 #(coll-psa/parse-value type %))
                                           (:values psa-ref))}]
      {:name (:name psa-ref)
       field-name (map (comp #(value->elastic-value type %)
                             #(coll-psa/parse-value type %))
                       (:values psa-ref))})))

(defn psa-refs->elastic-docs
  "Converts the psa-refs into a list of elastic documents"
  [collection granule]
  (let [parent-type-map (into {} (for [psa (:product-specific-attributes collection)]
                                   [(:name psa) (:data-type psa)]))]
    (map (fn [psa-ref]
           (let [type (parent-type-map (:name psa-ref))]
             (when-not type
               (errors/internal-error! (format "Could not find parent attribute [%s] in collection [%s]"
                                               (:name psa-ref)
                                               (:concept-id collection))))
             (psa-ref->elastic-doc type psa-ref)))
         (:product-specific-attributes granule))))

(defn psa->elastic-doc
  "Converts a PSA into the portion going in an elastic document"
  [psa]
  (let [{:keys [name data-type value]} psa
        field-name (type->field-name data-type)]
    (if (= :string data-type)
      [{:name name
        field-name (value->elastic-value data-type value)}
       {:name name
        (str field-name ".lowercase") (s/lower-case (value->elastic-value data-type value))}]
      {:name name
       field-name (value->elastic-value data-type value)})))

(defn psas->elastic-docs
  "Converts the psa into a list of elastic documents"
  [collection]<commit_msg>CMR-227: Added support for boolean, time-string, date-string, and datetime-string psa for collections.<commit_after>(ns cmr.indexer.data.concepts.attribute
  "Contains functions for converting attributes into a elastic documents"
  (:require [clj-time.format :as f]
            [clojure.string :as s]
            [cmr.umm.collection.product-specific-attribute :as coll-psa]
            [cmr.common.services.errors :as errors]))

(defmulti value->elastic-value
  "Converts a attribute value into the elastic value that should be transmitted"
  (fn [type value]
    type))

(defmethod value->elastic-value :default
  [type value]
  value)

(defmethod value->elastic-value :datetime
  [type value]
  ;; clj-time treats nil values as *now*. Adding explicit checks here to avoid weird errors.
  (when-not value (errors/internal-error! "Value is null"))
  (f/unparse (f/formatters :date-time) value))

(defmethod value->elastic-value :time
  [type value]
  ;; clj-time treats nil values as *now*. Adding explicit checks here to avoid weird errors.
  (when-not value (errors/internal-error! "Value is null"))
  ;; This relies on the fact that times are parsed into times on day 1970-01-01
  (f/unparse (f/formatters :date-time) value))

(defmethod value->elastic-value :date
  [type value]
  ;; clj-time treats nil values as *now*. Adding explicit checks here to avoid weird errors.
  (when-not value (errors/internal-error! "Value is null"))
  (f/unparse (f/formatters :date-time) value))

(def type->field-name
  "Converts an attribute type into the indexed field name"
  {:string "string-value"
   :time "time-value"
   :date "date-value"
   :datetime "datetime-value"
   :boolean "string-value"
   :time-string "string-value"
   :date-string "string-value"
   :datetime-string "string-value"
   :int "int-value"
   :float "float-value"})

(defn psa-ref->elastic-doc
  "Converts a PSA ref into the portion going in an elastic document"
  [type psa-ref]
  (let [field-name (type->field-name type)]
    (if (some #{type} [:string :boolean :time-string :date-string :datetime-string])
      [{:name (:name psa-ref)
        field-name (map (comp #(value->elastic-value type %)
                              #(coll-psa/parse-value type %))
                        (:values psa-ref))}
       {:name (:name psa-ref)
        (str field-name ".lowercase") (map (comp s/lower-case
                                                 #(value->elastic-value type %)
                                                 #(coll-psa/parse-value type %))
                                           (:values psa-ref))}]
      {:name (:name psa-ref)
       field-name (map (comp #(value->elastic-value type %)
                             #(coll-psa/parse-value type %))
                       (:values psa-ref))})))

(defn psa-refs->elastic-docs
  "Converts the psa-refs into a list of elastic documents"
  [collection granule]
  (let [parent-type-map (into {} (for [psa (:product-specific-attributes collection)]
                                   [(:name psa) (:data-type psa)]))]
    (map (fn [psa-ref]
           (let [type (parent-type-map (:name psa-ref))]
             (when-not type
               (errors/internal-error! (format "Could not find parent attribute [%s] in collection [%s]"
                                               (:name psa-ref)
                                               (:concept-id collection))))
             (psa-ref->elastic-doc type psa-ref)))
         (:product-specific-attributes granule))))

(defn psa->elastic-doc
  "Converts a PSA into the portion going in an elastic document"
  [psa]
  (let [{:keys [name data-type value]} psa
        field-name (type->field-name data-type)]
    (if (some #{data-type} [:string :boolean :time-string :date-string :datetime-string])
      [{:name name
        field-name (value->elastic-value data-type value)}
       {:name name
        (str field-name ".lowercase") (s/lower-case (value->elastic-value data-type value))}]
      {:name name
       field-name (value->elastic-value data-type value)})))

(defn psas->elastic-docs
  "Converts the psa into a list of elastic documents"
  [collection]
{'fraction_non_alphanumeric': 0.13173065129400222, 'fraction_numerical': 0.002330430516374341, 'mean_word_length': 2.339066339066339, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5516022', 'n_tokens_mistral': 2525, 'n_tokens_neox': 2388, 'n_words': 735}