<commit_before>(ns meins.jvm.graphql.custom-fields
  (:require [meins.jvm.graph.query :as gq]
            [taoensso.timbre :refer [info error warn debug]]
            [matthiasn.systems-toolbox.component :as stc]
            [ubergraph.core :as uber]
            [meins.jvm.datetime :as dt]
            [clj-time.format :as ctf]
            [clj-time.core :as ct]
            [meins.jvm.graphql.common :as gc]))

(defn custom-fields-cfg
  "Generates the custom fields config map as required by the
   user interface. The usage of custom fields in the UI predates the
   definition of custom fields in a specialized entry. The data
   format should be adjusted subsequently."
  [state]
  (debug "custom-fields-cfg")
  (let [q {:tags #{"#custom-field-cfg"}
           :n    Integer/MAX_VALUE}
        res (:entries-list (gq/get-filtered state q))
        f (fn [entry]
            (let [{:keys [tag items pvt active]} (:custom_field_cfg entry)
                  story (:primary_story entry)
                  fm (fn [field]
                       (let [k (keyword (:name field))
                             label (:label field)]
                         [k {:cfg   field
                             :label label}]))
                  fields (into {} (map fm items))]
              [tag {:default-story story
                    :timestamp     (:timestamp entry)
                    :adjusted_ts   (:adjusted_ts entry)
                    :pvt           pvt
                    :active        active
                    :fields        fields}]))
        res (->> (map f res)
                 (sort-by #(or (:adjusted_ts (second %))
                               (:timestamp (second %))))
                 (filter first)
                 reverse
                 (into {}))]
    (debug "custom-fields-cfg" res)
    res))


(def dtz (ct/default-time-zone))
(def fmt (ctf/formatter "yyyy-MM-dd'T'HH:mm" dtz))
(defn parse [dt] (ctf/parse fmt dt))

(defn bool-conv [x]
  (if (boolean? x)
    (if x 1 0)
    x))

(defn val-mapper [k field entry]
  (let [path [:custom_fields k field]
        ts (or (:adjusted_ts entry)
               (:timestamp entry))]
    {:v  (bool-conv (get-in entry path))
     :ts ts}))

(defn stats-mapper [tag nodes [k fields]]
  (let [field-mapper
        (fn [[field v]]
          (let [op (when (contains? #{:number :time :switch} (:type (:cfg v)))
                     (case (:agg (:cfg v))
                       :min #(when (seq %) (apply min (map :v %)))
                       :max #(when (seq %) (apply max (map :v %)))
                       :mean #(when (seq %) (double (/ (apply + (map :v %)) (count %))))
                       #(apply + (map :v %))))
                res (vec (filter #(:v %) (mapv (partial val-mapper k field) nodes)))]
            [field {:v   (if op
                           (try (op res)
                                (catch Exception e (error e res)))
                           res)
                    :vs  res
                    :tag tag}]))]
    (into {} (mapv field-mapper fields))))

(defn adjusted-ts-filter [date-string entry]
  (let [adjusted-ts (:adjusted_ts entry)
        tz (:timezone entry)]
    (or (not adjusted-ts)
        (= (dt/ts-to-ymd-tz adjusted-ts tz)
           date-string))))

(defn fields-mapper [[k {:keys [v tag vs]}]]
  {:field  (name k)
   :tag    tag
   :value  (bool-conv v)
   :values vs})

(defn custom-fields-mapper
  "Creates mapper function for custom field stats. Takes current state. Returns
   function that takes date string, such as '2016-10-10', and returns map with
   results for the defined custom fields, plus the date string. Performs
   operation specified for field, such as sum, min, max."
  [current-state tag]
  (let [custom-fields (custom-fields-cfg current-state)
        fields-def (into {} (map (fn [[k v]] [k (:fields v)])
                                 (select-keys custom-fields [tag])))
        g (:graph current-state)]
    (fn [date-string]
      (let [day-nodes (gq/get-nodes-for-day g {:date_string date-string})
            day-nodes-attrs (map #(uber/attrs g %) day-nodes)
            nodes (filter :custom_fields day-nodes-attrs)
            nodes (filter (partial adjusted-ts-filter date-string) nodes)
            fields (mapv (partial stats-mapper tag nodes) fields-def)]
        (apply merge
               {:date_string date-string
                :fields      (mapv fields-mapper (first fields))
                :tag         tag}
               fields)))))

(defn custom-fields-mapper2
  "Creates mapper function for custom field stats. Takes current state. Returns
   function that takes date string, such as '2016-10-10', and returns map with
   results for the defined custom fields, plus the date string. Performs
   operation specified for field, such as sum, min, max."
  [current-state tag]
  (let [custom-fields (-> current-state :options :custom_fields)
        fields-def (into {} (map (fn [[k v]] [k (:fields v)])
                                 (select-keys custom-fields [tag])))]
    (fn [day nodes]
      (let [nodes (filter :custom_fields nodes)
            nodes (filter (partial adjusted-ts-filter day) nodes)
            fields (mapv (partial stats-mapper tag nodes) fields-def)]
        (apply merge
               {:date_string day
                :fields      (mapv fields-mapper (first fields))
                :tag         tag}
               fields)))))<commit_msg>fix customfield query in habit stats<commit_after>(ns meins.jvm.graphql.custom-fields
  (:require [meins.jvm.graph.query :as gq]
            [taoensso.timbre :refer [info error warn debug]]
            [matthiasn.systems-toolbox.component :as stc]
            [ubergraph.core :as uber]
            [meins.jvm.datetime :as dt]
            [clj-time.format :as ctf]
            [clj-time.core :as ct]
            [meins.jvm.graphql.common :as gc]))

(defn custom-fields-cfg
  "Generates the custom fields config map as required by the
   user interface. The usage of custom fields in the UI predates the
   definition of custom fields in a specialized entry. The data
   format should be adjusted subsequently."
  [state]
  (debug "custom-fields-cfg")
  (let [q {:tags #{"#custom-field-cfg"}
           :n    Integer/MAX_VALUE}
        res (:entries-list (gq/get-filtered state q))
        f (fn [entry]
            (let [{:keys [tag items pvt active]} (:custom_field_cfg entry)
                  story (:primary_story entry)
                  fm (fn [field]
                       (let [k (keyword (:name field))
                             label (:label field)]
                         [k {:cfg   field
                             :label label}]))
                  fields (into {} (map fm items))]
              [tag {:default-story story
                    :timestamp     (:timestamp entry)
                    :adjusted_ts   (:adjusted_ts entry)
                    :pvt           pvt
                    :active        active
                    :fields        fields}]))
        res (->> (map f res)
                 (sort-by #(or (:adjusted_ts (second %))
                               (:timestamp (second %))))
                 (filter first)
                 reverse
                 (into {}))]
    (debug "custom-fields-cfg" res)
    res))


(def dtz (ct/default-time-zone))
(def fmt (ctf/formatter "yyyy-MM-dd'T'HH:mm" dtz))
(defn parse [dt] (ctf/parse fmt dt))

(defn bool-conv [x]
  (if (boolean? x)
    (if x 1 0)
    x))

(defn val-mapper [k field entry]
  (let [path [:custom_fields k field]
        ts (or (:adjusted_ts entry)
               (:timestamp entry))]
    {:v  (bool-conv (get-in entry path))
     :ts ts}))

(defn stats-mapper [tag nodes [k fields]]
  (let [field-mapper
        (fn [[field v]]
          (let [op (when (contains? #{:number :time :switch} (:type (:cfg v)))
                     (case (:agg (:cfg v))
                       :min #(when (seq %) (apply min (map :v %)))
                       :max #(when (seq %) (apply max (map :v %)))
                       :mean #(when (seq %) (double (/ (apply + (map :v %)) (count %))))
                       #(apply + (map :v %))))
                res (vec (filter #(:v %) (mapv (partial val-mapper k field) nodes)))]
            [field {:v   (if op
                           (try (op res)
                                (catch Exception e (error e res)))
                           res)
                    :vs  res
                    :tag tag}]))]
    (into {} (mapv field-mapper fields))))

(defn adjusted-ts-filter [date-string entry]
  (let [adjusted-ts (:adjusted_ts entry)
        tz (:timezone entry)]
    (or (not adjusted-ts)
        (= (dt/ts-to-ymd-tz adjusted-ts tz)
           date-string))))

(defn fields-mapper [[k {:keys [v tag vs]}]]
  {:field  (name k)
   :tag    tag
   :value  (bool-conv v)
   :values vs})

(defn custom-fields-mapper
  "Creates mapper function for custom field stats. Takes current state. Returns
   function that takes date string, such as '2016-10-10', and returns map with
   results for the defined custom fields, plus the date string. Performs
   operation specified for field, such as sum, min, max."
  [current-state tag]
  (let [custom-fields (custom-fields-cfg current-state)
        fields-def (into {} (map (fn [[k v]] [k (:fields v)])
                                 (select-keys custom-fields [tag])))
        g (:graph current-state)]
    (fn [date-string]
      (let [day-nodes (gq/get-nodes-for-day g {:date_string date-string})
            day-nodes-attrs (map #(uber/attrs g %) day-nodes)
            nodes (filter :custom_fields day-nodes-attrs)
            nodes (filter (partial adjusted-ts-filter date-string) nodes)
            fields (mapv (partial stats-mapper tag nodes) fields-def)]
        (apply merge
               {:date_string date-string
                :fields      (mapv fields-mapper (first fields))
                :tag         tag}
               fields)))))

(defn custom-fields-mapper2
  "Creates mapper function for custom field stats. Takes current state. Returns
   function that takes date string, such as '2016-10-10', and returns map with
   results for the defined custom fields, plus the date string. Performs
   operation specified for field, such as sum, min, max."
  [current-state tag]
  (let [custom-fields (-> current-state :cfg :custom-fields)
        fields-def (into {} (map (fn [[k v]] [k (:fields v)])
                                 (select-keys custom-fields [tag])))]
    (fn [day nodes]
      (let [nodes (filter :custom_fields nodes)
            nodes (filter (partial adjusted-ts-filter day) nodes)
            fields (mapv (partial stats-mapper tag nodes) fields-def)]
        (apply merge
               {:date_string day
                :fields      (mapv fields-mapper (first fields))
                :tag         tag}
               fields)))))
{'fraction_non_alphanumeric': 0.12510341023991176, 'fraction_numerical': 0.0034929681036860007, 'mean_word_length': 1.7017630990812018, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5820769', 'n_tokens_mistral': 3289, 'n_tokens_neox': 3124, 'n_words': 1046}