<commit_before>(ns ow.factum.facts
  (:require [ow.factum.db :as db]))

(defn projected-facts [eventstore & {:keys [timestamp]}]
  "Projects raw facts to a given timestamp.
This effectively filters those facts that are relevant for the given timestamp,
i.e. it removes obsolete old facts that are overriden by newer ones or have been
retracted later on (but before timestamp). If no timestamp is given, current time
is being assumed."
  (let [rawfacts (db/get-all eventstore)
        xf (fn [xf]
             (let [facts* (volatile! {})]
               (fn
                 ([] (xf))
                 ([result] (xf result))
                 ([result [e a v t action :as input]]
                  (case action
                    :add (case (get-in @facts* [e a])
                           true result
                           ::retracted (do (vswap! facts* #(update-in % [e] dissoc a))
                                           result)
                           nil (do (vswap! facts* #(assoc-in % [e a] true))
                                   (xf result (take 4 input))))
                    :retract (do (vswap! facts* #(assoc-in % [e a] ::retracted))
                                 result))))))]
    (into [] xf rawfacts)))

(defn add-facts [eventstore facts]
  "Adds one or more facts within one single transaction."
  (let [txid (db/new-txid eventstore)]
    (sequence (comp (map #(->> (concat % (repeat nil))
                               (take 5)
                               vec))<commit_msg>fix retracted facts handling<commit_after>(ns ow.factum.facts
  (:require [ow.factum.db :as db]))

(defn projected-facts [eventstore & {:keys [timestamp]}]
  "Projects raw facts to a given timestamp.
This effectively filters those facts that are relevant for the given timestamp,
i.e. it removes obsolete old facts that are overriden by newer ones or have been
retracted later on (but before timestamp). If no timestamp is given, current time
is being assumed."
  (let [rawfacts (db/get-all eventstore)
        xf (fn [xf]
             (let [facts* (volatile! {})]
               (fn
                 ([] (xf))
                 ([result] (xf result))
                 ([result [e a v t action :as input]]
                  (case action
                    :add (case (get-in @facts* [e a])
                           true result
                           ::retracted (do (vswap! facts* #(update-in % [e] dissoc a))
                                           result)
                           nil (do (vswap! facts* #(assoc-in % [e a] true))
                                   (xf result (take 4 input))))
                    :retract (do (when (nil? (get-in @facts* [e a]))
                                   (vswap! facts* #(assoc-in % [e a] ::retracted)))
                                 result))))))]
    (into [] xf rawfacts)))

(defn add-facts [eventstore facts]
  "Adds one or more facts within one single transaction."
  (let [txid (db/new-txid eventstore)]
    (sequence (comp (map #(->> (concat % (repeat nil))
                               (take 5)
                               vec))
{'fraction_non_alphanumeric': 0.11636129861780778, 'fraction_numerical': 0.0012857602057216328, 'mean_word_length': 1.4896, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6584079', 'n_tokens_mistral': 879, 'n_tokens_neox': 825, 'n_words': 316}