<commit_before>(ns onyx-java.utility.interop
  (:gen-class :name onyx-java.utility.interop
              :methods [^:static [write_batch [clojure.lang.IPersistentMap] clojure.lang.IPersistentMap]
                        ^:static [read_batch [clojure.lang.IPersistentMap] clojure.lang.IPersistentMap]])
  (:require [onyx.information-model :refer [model]]))

(defn -write_batch
  [event]
  ((resolve 'onyx.peer.function/write-batch) event))

(defn -read_batch
  [event]
  ((resolve 'onyx.peer.function/read-batch) event))

(gen-interface
  :name onyx-java.IPipeline
  :methods [[writeBatch [clojure.lang.IPersistentMap] clojure.lang.IPersistentMap]])

(defn boolean-string [y] (if (= (type y) java.lang.String) (read-string y) y))

(def casts
  {:boolean (fn [x] (boolean (boolean-string x)))
   :integer (fn [x] (Integer/parseInt (re-find #"\A-?\d+" x)))
   :string (fn [x] (str x))
   :any (fn [x] x)
   :keyword (fn [x] (keyword x))
   :vector (fn [x] (vec x))})

(defn check-choices [choices value]
    (if (boolean (some #{:all} (flatten choices))) value
        (if (some #{value} choices) value (throw (Exception. "Not a valid choice.")))))

(defn get-required-keymap [section]
    (let [required-keys (vec (keys (filter (fn [[k v]] (and (-> v :optional? not)
                                    (not (contains? v :deprecated-version))))
                                    (get-in model [section :model]))))]
     (apply merge (map #(hash-map % nil) required-keys))))

(defn update-required-keymap [required-keymap keyword]
    ;(println keyword)
    (if (boolean (some #{keyword} (keys required-keymap))) (assoc required-keymap keyword true) required-keymap))

(defn cast-types [section m]
  (let [section* (keyword section)]
        ;required-keymap (get-required-keymap section*)]
    (reduce-kv
     (fn [m* k v]
       (let [k* (keyword k)
             type (get-in model [section* :model k* :type])
             choices (get-in model [section* :model k* :choices] [:all])
             required (not (get-in model [section* :model k* :optional?] true))
             v* (check-choices choices ((get casts type identity) v))]
         (assoc m* k* v*)))
     {}
     m)))

(defn coerce-workflow [workflow]
  (mapv #(mapv (fn [v] (keyword v)) %) workflow))

(defn coerce-catalog [catalog]
  (mapv #(cast-types :catalog-entry %) catalog))
<commit_msg>Now returns a map if there's an error in missing required keys or incorrect choice.<commit_after>(ns onyx-java.utility.interop
  (:gen-class :name onyx-java.utility.interop
              :methods [^:static [write_batch [clojure.lang.IPersistentMap] clojure.lang.IPersistentMap]
                        ^:static [read_batch [clojure.lang.IPersistentMap] clojure.lang.IPersistentMap]])
  (:require [onyx.information-model :refer [model]]))

(defn -write_batch
  [event]
  ((resolve 'onyx.peer.function/write-batch) event))

(defn -read_batch
  [event]
  ((resolve 'onyx.peer.function/read-batch) event))

(gen-interface
  :name onyx-java.IPipeline
  :methods [[writeBatch [clojure.lang.IPersistentMap] clojure.lang.IPersistentMap]])

(defn boolean-string [x]
    (if (= (type x) java.lang.String) (read-string x) x))

(def casts
  {:boolean (fn [x] (boolean (boolean-string x)))
   :integer (fn [x] (Integer/parseInt (re-find #"\A-?\d+" x)))
   :string (fn [x] (str x))
   :any (fn [x] x)
   :keyword (fn [x] (keyword x))
   :vector (fn [x] (vec x))})

(defn check-choices [choices value]
    (if (boolean (some #{:all} (flatten choices))) value
        (if (some #{value} choices)
            value
            "CHOICE-ERROR"
            ;(throw (Exception. "Not a valid choice."))
            )))

(defn get-required-keymap [section]
    (let [required-keys (vec (keys (filter (fn [[k v]] (and (-> v :optional? not)
                                    (not (contains? v :deprecated-version))))
                                    (get-in model [section :model]))))]
     (apply merge (map #(hash-map % nil) required-keys))))

(defn update-required-keymap [required-keymap keyword]
    (if (boolean (some #{keyword} (keys required-keymap)))
        (assoc required-keymap keyword true)
        required-keymap))

(defn required-keywords [required existing]
    (let [missing (clojure.set/difference (set (keys required)) (set (keys existing)))]
    (if (= 0 (count missing))
        nil
        missing)))

(defn cast-types [section m]
  (let [section* (keyword section)
        required-keymap (get-required-keymap section*)
        new-map (reduce-kv
                    (fn [m* k v]
                        (let [k* (keyword k)
                        type (get-in model [section* :model k* :type])
                        choices (get-in model [section* :model k* :choices] [:all])
                        v* (check-choices choices ((get casts type identity) v))]
                        (assoc m* k* v*)))
                    {}
                    m)]
    (def missing (required-keywords required-keymap new-map))
    (if (not missing)
        new-map
        (apply merge (map #(hash-map % "KEY-ERROR") missing))
    )))

(defn coerce-workflow [workflow]
  (mapv #(mapv (fn [v] (keyword v)) %) workflow))

(defn coerce-catalog [catalog]
  (mapv #(cast-types :catalog-entry %) catalog))

{'fraction_non_alphanumeric': 0.15885714285714286, 'fraction_numerical': 0.00019047619047619048, 'mean_word_length': 2.6288873531444366, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1988234', 'n_tokens_mistral': 1793, 'n_tokens_neox': 1755, 'n_words': 485}