<commit_before>(ns cfpb.qu.data.aggregation
  "This namespace contains functions for generating aggregations
within Mongo."
  (:require
   [cfpb.qu.data.compression :refer [compress-where field-zip-fn]]
   [cfpb.qu.util :refer :all]
   [taoensso.timbre :as log]   
   [com.reasonr.scriptjure :refer [cljs cljs* js js*]]))

(def ^:private agg-fns
  {:min (js* (fn [ary field]
               (var vals (.map ary (fn [obj] (return (aget obj field)))))
               (return (.apply (aget Math "min") nil vals))))
   :max (js* (fn [ary field]
               (var vals (.map ary (fn [obj] (return (aget obj field)))))
               (return (.apply (aget Math "max") nil vals))))
   :avg (js* (fn [ary field]
               (var vals (.map ary (fn [obj] (return (aget obj field)))))
               (return (.avg Array vals))))
   :median (js* (fn [ary field]
                  (var median (fn [array]
                                (var array (.sort array))
                                (var len (aget array "length"))
                                (if (= len 0)
                                  (return nil))
                                (if (= (% len 2) 0)
                                  (return (/ (+ (aget array (- 1 (/ len 2)))
                                                (aget array (/ len 2)))
                                             2))
                                  (return (aget array (/ (- len 1) 2))))))
                  (var vals (.map ary (fn [obj] (return (aget obj field)))))
                  (return (median vals))))
   :sum (js* (fn [ary field]
               (var vals (.map ary (fn [obj] (return (aget obj field)))))
               (return (.sum Array vals))))
   :count (js* (fn [ary field]
                 (var vals (.map ary (fn [obj] (return (aget obj field)))))
                 (return (.sum Array vals))))})

(defn- generate-map-fn
  ([group-fields agg-fields]
     (generate-map-fn group-fields agg-fields identity))
  ([group-fields agg-fields zip-fn]
     (let [create-map-obj (fn [fields]
                            (reduce (fn [acc [out-field in-field]]
                                      (merge acc {out-field
                                                  (if (number? in-field)
                                                    in-field
                                                    (let [from-field (name (zip-fn in-field))]
                                                      (js* (aget this (clj from-field)))))}))
                                    {} fields))
           map-id (create-map-obj group-fields)
           map-val (create-map-obj agg-fields)]
       (js* (fn [] (emit (clj map-id) (clj map-val)))))))

(defn- generate-reduce-fn
  [aggregations]
  (let [reduce-obj (reduce (fn [acc [out-field [agg in-field]]]
                             (merge acc
                                    {(name out-field)
                                     (js* ((aget aggs (clj agg)) values (clj (name out-field))))}))
                           {} aggregations)]
    (js* (fn [key values]
           (var aggs (clj agg-fns))
           (return (clj reduce-obj))))))

(defn generate-map-reduce*
  "Generate a map with the information needed to run map-reduce inside
  Mongo. This does not convert the map and reduce functions to strings
  with JS in them, so it works for generating something you can paste
  right into Mongo, like so:

  (cljs (generate-map-reduce* {:dataset \"integration_test\"
                               :from \"incomes\"
                               :to \"test1\"
                               :group [:state_abbr]
                               :aggregations {:max_tax_returns [\"max\" \"tax_returns\"]}
                               :slicedef from-slice-definition}))

  If you want map and reduce already stringified, to send through
  Monger, run `generate-map-reduce`."  
  [{:keys [dataset from to group aggregations filter slicedef]}]
  {:pre [(every? #(not (nil? %)) [dataset from to group aggregations])
         (sequential? group)]}
  (let [field-zip-fn (if slicedef
                       (field-zip-fn slicedef)
                       identity)
        query (if filter
                (-> filter
                    (compress-where field-zip-fn)
                    (convert-keys name))
                {})<commit_msg>Ensure that aggregations without aggregated values do not fail<commit_after>(ns cfpb.qu.data.aggregation
  "This namespace contains functions for generating aggregations
within Mongo."
  (:require
   [cfpb.qu.data.compression :refer [compress-where field-zip-fn]]
   [cfpb.qu.util :refer :all]
   [taoensso.timbre :as log]   
   [com.reasonr.scriptjure :refer [cljs cljs* js js*]]))

(def ^:private agg-fns
  {:min (js* (fn [ary field]
               (var vals (.map ary (fn [obj] (return (aget obj field)))))
               (return (.apply (aget Math "min") nil vals))))
   :max (js* (fn [ary field]
               (var vals (.map ary (fn [obj] (return (aget obj field)))))
               (return (.apply (aget Math "max") nil vals))))
   :avg (js* (fn [ary field]
               (var vals (.map ary (fn [obj] (return (aget obj field)))))
               (return (.avg Array vals))))
   :median (js* (fn [ary field]
                  (var median (fn [array]
                                (var array (.sort array))
                                (var len (aget array "length"))
                                (if (= len 0)
                                  (return nil))
                                (if (= (% len 2) 0)
                                  (return (/ (+ (aget array (- 1 (/ len 2)))
                                                (aget array (/ len 2)))
                                             2))
                                  (return (aget array (/ (- len 1) 2))))))
                  (var vals (.map ary (fn [obj] (return (aget obj field)))))
                  (return (median vals))))
   :sum (js* (fn [ary field]
               (var vals (.map ary (fn [obj] (return (aget obj field)))))
               (return (.sum Array vals))))
   :count (js* (fn [ary field]
                 (var vals (.map ary (fn [obj] (return (aget obj field)))))
                 (return (.sum Array vals))))})

(defn- generate-map-fn
  ([group-fields agg-fields]
     (generate-map-fn group-fields agg-fields identity))
  ([group-fields agg-fields zip-fn]
     (let [create-map-obj (fn [fields]
                            (reduce (fn [acc [out-field in-field]]
                                      (merge acc {out-field
                                                  (if (number? in-field)
                                                    in-field
                                                    (let [from-field (name (zip-fn in-field))]
                                                      (js* (aget this (clj from-field)))))}))
                                    {} fields))
           map-id (create-map-obj group-fields)
           map-val (create-map-obj agg-fields)]
       (js* (fn [] (emit (clj map-id) (clj map-val)))))))

(defn- generate-reduce-fn
  [aggregations]
  (let [reduce-obj (reduce (fn [acc [out-field [agg in-field]]]
                             (merge acc
                                    {(name out-field)
                                     (js* ((aget aggs (clj agg)) values (clj (name out-field))))}))
                           {} aggregations)]
    (js* (fn [key values]
           (var aggs (clj agg-fns))
           (return (clj reduce-obj))))))

(defn generate-map-reduce*
  "Generate a map with the information needed to run map-reduce inside
  Mongo. This does not convert the map and reduce functions to strings
  with JS in them, so it works for generating something you can paste
  right into Mongo, like so:

  (cljs (generate-map-reduce* {:dataset \"integration_test\"
                               :from \"incomes\"
                               :to \"test1\"
                               :group [:state_abbr]
                               :aggregations {:max_tax_returns [\"max\" \"tax_returns\"]}
                               :slicedef from-slice-definition}))

  If you want map and reduce already stringified, to send through
  Monger, run `generate-map-reduce`."  
  [{:keys [dataset from to group aggregations filter slicedef] :or {aggregations {}}}]
  {:pre [(every? #(not (nil? %)) [dataset from to group aggregations])
         (sequential? group)]}
  (let [field-zip-fn (if slicedef
                       (field-zip-fn slicedef)
                       identity)
        query (if filter
                (-> filter
                    (compress-where field-zip-fn)
                    (convert-keys name))
                {})
{'fraction_non_alphanumeric': 0.12872644203312392, 'fraction_numerical': 0.002284408909194746, 'mean_word_length': 1.2945492662473794, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 46, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5503908', 'n_tokens_mistral': 2486, 'n_tokens_neox': 2336, 'n_words': 776}