<commit_before>(ns hface.refresh
  (:require [clojure.tools.logging :refer [info]]
            [hface.util :refer [every]]
            [hface.stats :refer [cluster-stats]]))

(def stats (atom {}))

(defn rate-it [previous current k rate-key interval]
    (let [rate (float (/ (- (k current) (k previous)) interval))]
      (if-not (neg? rate)        ;; can be negative if one of the nodes parted
        rate
        (rate-key previous))))   ;; in which case return a previous rate

;; calculating rates for maps (e.g. hazelcast IMap/MultiMap)

(defn rate-map [pm cm interval]
  (assoc cm :get-rate (rate-it pm cm :get-count :get-rate interval)
            :put-rate (rate-it pm cm :put-count :put-rate interval)
            :hit-rate (rate-it pm cm :hits :hit-rate interval)))

(defn compare-and-rate-maps [previous current interval]
  (into {}
    (for [[k v] current]
      (if-let [pv (k previous)]           ;; if there is a value in a prev map for this key
        [k (rate-map pv v interval)]
        [k (rate-map v v interval)]))))

(defn add-map-ratings [p-stats            ;; previous stats
                       c-stats            ;; current stats
                       interval
                       mtype]             ;; :map-stats, :multi-map-stats
  (let [m-stats #(-> % :aggregated mtype)
        previous (m-stats p-stats)
        current (m-stats c-stats)]
    (assoc-in c-stats [:aggregated mtype]
              (compare-and-rate-maps previous current interval))))

;;-----------------------------------------------------------------

(defn refresh-stats 
  "refreshing stats and adding rate metrics"
  [previous interval]
  (let [current (cluster-stats)
        with-map-rates (add-map-ratings @previous current interval :map-stats)
        with-rates (add-map-ratings @previous with-map-rates interval :multi-map-stats)]     ;; TODO: refactor out and add other ratings
    (reset! previous with-rates)))

(defn collect-stats 
  "scheduling stats refresh on every 'interval'"
  [interval]
  (every interval #(refresh-stats stats interval)))
<commit_msg>adding rates to q-stats<commit_after>(ns hface.refresh
  (:require [clojure.tools.logging :refer [info]]
            [hface.util :refer [every]]
            [hface.stats :refer [cluster-stats]]))

(def stats (atom {}))

(defn rate-it [previous current k rate-key interval]
    (let [rate (float (/ (- (k current) (k previous)) interval))]
      (if-not (neg? rate)        ;; can be negative if one of the nodes parted
        rate
        (rate-key previous))))   ;; in which case return a previous rate

;; calculating rates for maps (e.g. hazelcast IMap/MultiMap)

(defn rate-map [pm cm interval]
  (assoc cm :get-rate (rate-it pm cm :get-count :get-rate interval)
            :put-rate (rate-it pm cm :put-count :put-rate interval)
            :hit-rate (rate-it pm cm :hits :hit-rate interval)))

(defn rate-q [pm cm interval]
  (assoc cm :take-rate (rate-it pm cm :number-of-offers :take-rate interval)
            :put-rate (rate-it pm cm :number-of-polls :put-rate interval)
            :rejected-put-rate (rate-it pm cm :number-of-rejected-offers :rejected-put-rate interval)
            :empty-take-rate (rate-it pm cm :number-of-empty-polls :empty-take-rate interval)))

(defn compare-and-rate [rate previous current interval]
  (into {}
    (for [[k v] current]
      (if-let [pv (k previous)]          ;; if there is a value in a prev map for this key
        [k (rate pv v interval)]
        [k (rate v v interval)]))))

(defn add-ratings [c-stats               ;; current stats
                   p-stats               ;; previous stats
                   rate
                   interval
                   s-type]               ;; :map-stats, :multi-map-stats, :queue-stats
  (let [stats #(-> % :aggregated s-type)
        previous (stats p-stats)
        current (stats c-stats)]
    (assoc-in c-stats [:aggregated s-type]
              (compare-and-rate rate previous current interval))))

;;-----------------------------------------------------------------

(defn refresh-stats 
  "refreshing stats and adding rate metrics"
  [previous interval]
  (let [with-rates (-> (cluster-stats)
                       (add-ratings @previous rate-map interval :map-stats)
                       (add-ratings @previous rate-map interval :multi-map-stats)
                       (add-ratings @previous rate-q interval :queue-stats))]
    (reset! previous with-rates)))

(defn collect-stats 
  "scheduling stats refresh on every 'interval'"
  [interval]
  (every interval #(refresh-stats stats interval)))

{'fraction_non_alphanumeric': 0.15216917375190755, 'fraction_numerical': 0.0, 'mean_word_length': 2.085406859448554, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4195529', 'n_tokens_mistral': 1390, 'n_tokens_neox': 1348, 'n_words': 442}