<commit_before>(ns mikron.test-util
  (:require [clojure.walk :as walk]
            [mikron.runtime.processor.validate :as runtime.processor.validate]))

(defn nan?
  "Returns `true` if value is NaN, `false otherwise`."
  [value]
  #?(:clj  (Double/isNaN value)
     :cljs (js/isNaN value)))

#?(:cljs
   (defn arraybuffer->seq
     "Converts an ArrayBuffer `value` to a sequence."
     [value]
     (seq (.from js/Array (js/Int8Array. value)))))

(defn fix-for-equality*
  "Returns a more 'equal friendly' value, if necessary."
  [value]
  (condp contains? (type value)
    #{runtime.processor.validate/binary-type}
    [:mikron/binary #?(:clj  (seq value)
                       :cljs (arraybuffer->seq value))]

    #?(:clj  #{java.lang.Double java.lang.Float}
       :cljs #{js/Number})
    (if (nan? value)
      :mikron/nan
      (double value))

    value))

(defn fix-for-equality
  "Returns a more 'equal friendly' value, if necessary."
  [value]
  (walk/postwalk fix-for-equality* value))

(defn equal?
  "Checks whether the given values are equal."
  [value-1 value-2]
  (= (fix-for-equality value-1)
     (fix-for-equality value-2)))<commit_msg>Rename `fix-for-equality` to `clean`<commit_after>(ns mikron.test-util
  (:require [clojure.walk :as walk]
            [mikron.runtime.processor.validate :as runtime.processor.validate]))

(defn nan?
  "Returns `true` if value is NaN, `false otherwise`."
  [value]
  #?(:clj  (Double/isNaN value)
     :cljs (js/isNaN value)))

#?(:cljs
   (defn arraybuffer->seq
     "Converts an ArrayBuffer `value` to a sequence."
     [value]
     (seq (.from js/Array (js/Int8Array. value)))))

(defn clean*
  "Returns a more 'equal friendly' value, if necessary."
  [value]
  (condp contains? (type value)
    #{runtime.processor.validate/binary-type}
    [:mikron/binary #?(:clj  (seq value)
                       :cljs (arraybuffer->seq value))]

    #?(:clj  #{java.lang.Double java.lang.Float}
       :cljs #{js/Number})
    (if (nan? value)
      :mikron/nan
      (double value))

    value))

(defn clean
  "Returns a more 'equal friendly' value, if necessary."
  [value]
  (walk/postwalk clean* value))

(defn equal?
  "Checks whether the given values are equal."
  [value-1 value-2]
  (= (clean value-1)
     (clean value-2)))
{'fraction_non_alphanumeric': 0.15687992988606486, 'fraction_numerical': 0.0043821209465381246, 'mean_word_length': 2.949826989619377, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1063382', 'n_tokens_mistral': 821, 'n_tokens_neox': 810, 'n_words': 214}