<commit_before>(ns cavm.query.functions
  (:use clojure.core.matrix)
  (:use clojure.core.matrix.operators)
  (:refer-clojure :exclude  [* - + == /])
  (:gen-class))

(set-current-implementation :vectorz)

(defn meannan1d [m]
  (let [NaN Double/NaN
        [sum n] (ereduce
                  (fn [[acc cnt] x] (if (Double/isNaN x) [acc cnt] [(+ acc x) (inc cnt)]))
                  [0 0]
                  m)]
    (if (= 0 n) NaN (/ sum n))))

; XXX this handling of dim is wrong for dim > 1
; XXX do we need to fill nan values, like we do in python?
(defn meannan [m dim]
    (let [new-shape (assoc (vec (shape m)) (long dim) 1)]
      (reshape (matrix (map meannan1d (slices m (- 1 dim)))) new-shape)))

(def functions
  {'+ +
   '/ /
   '* emul
   '- -
   'mean meannan
   'apply apply})<commit_msg>Add boolean operators. Still need to extend these for matrices.<commit_after>(ns cavm.query.functions
  (:use clojure.core.matrix)
  (:use clojure.core.matrix.operators)
  (:refer-clojure :exclude  [* - + == /])
  (:gen-class))

(set-current-implementation :vectorz)

(defn meannan1d [m]
  (let [NaN Double/NaN
        [sum n] (ereduce
                  (fn [[acc cnt] x] (if (Double/isNaN x) [acc cnt] [(+ acc x) (inc cnt)]))
                  [0 0]
                  m)]
    (if (= 0 n) NaN (/ sum n))))

; XXX this handling of dim is wrong for dim > 1
; XXX do we need to fill nan values, like we do in python?
(defn meannan [m dim]
    (let [new-shape (assoc (vec (shape m)) (long dim) 1)]
      (reshape (matrix (map meannan1d (slices m (- 1 dim)))) new-shape)))

(def functions
  {'+ +
   '/ /
   '* emul
   '> >
   '< <
   '>= >=
   '<= <=
   '- -
   '= ==
   'mean meannan
   'apply apply}) ; XXX is this a security hole? Can the user pass in clojure symbols?
{'fraction_non_alphanumeric': 0.1655367231638418, 'fraction_numerical': 0.00903954802259887, 'mean_word_length': 2.297951582867784, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4012387', 'n_tokens_mistral': 679, 'n_tokens_neox': 647, 'n_words': 209}