<commit_before>(ns abracad.avro.read
  "Generic data reading implementation."
  {:private true}
  (:require [abracad.avro.util :refer [mangle unmangle field-keyword]]
            [abracad.avro :as avro])
  (:import [org.apache.avro Schema Schema$Field]
           [org.apache.avro.io Decoder ResolvingDecoder]
           [abracad.avro ClojureDatumReader]))

(defn schema-symbol
  [^Schema schema]
  (let [ns (.getNamespace schema), n (-> schema .getName unmangle)]
    (if ns (symbol (unmangle ns) n) (symbol n))))

(defn reader-fn
  [^Schema schema rname]
  (or (get avro/*avro-readers* rname)
      (if-let [reader (.getProp schema "abracad.reader")]
        (case reader
          nil      nil
          "vector" vector
          #_else   (-> reader symbol resolve)))))

(defn read-record
  [^ClojureDatumReader reader ^Schema expected ^ResolvingDecoder in]
  (let [rname (schema-symbol expected)
        readerf (reader-fn expected rname)
        [reducef record] (if readerf
                           [(fn [v ^Schema$Field f]
                              (conj! v (.read reader nil (.schema f) in)))<commit_msg>Arbitrary code-execution from schemas is probably bad.<commit_after>(ns abracad.avro.read
  "Generic data reading implementation."
  {:private true}
  (:require [abracad.avro.util :refer [mangle unmangle field-keyword]]
            [abracad.avro :as avro])
  (:import [org.apache.avro Schema Schema$Field]
           [org.apache.avro.io Decoder ResolvingDecoder]
           [abracad.avro ClojureDatumReader]))

(defn schema-symbol
  [^Schema schema]
  (let [ns (.getNamespace schema), n (-> schema .getName unmangle)]
    (if ns (symbol (unmangle ns) n) (symbol n))))

(defn reader-fn
  [^Schema schema rname]
  (or (get avro/*avro-readers* rname)
      (if-let [reader (.getProp schema "abracad.reader")]
        (case reader
          "vector" vector
          #_else   (throw (ex-info "unknown `abracad.reader`"
                                   {:reader reader}))))))

(defn read-record
  [^ClojureDatumReader reader ^Schema expected ^ResolvingDecoder in]
  (let [rname (schema-symbol expected)
        readerf (reader-fn expected rname)
        [reducef record] (if readerf
                           [(fn [v ^Schema$Field f]
                              (conj! v (.read reader nil (.schema f) in)))
{'fraction_non_alphanumeric': 0.12435456110154905, 'fraction_numerical': 0.0, 'mean_word_length': 2.5015060240963853, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '502573', 'n_tokens_mistral': 747, 'n_tokens_neox': 722, 'n_words': 210}