<commit_before>(ns moebius.kv
  (:refer-clojure :exclude [set get remove update]))

;;
;; TODO: multi set function
;; TODO: force keys to string?
;; TODO: Do ve need the version? (think repartitioning)
;; TOOD: tx-log + compaction: single key or all keys x sourceId?
;;

;;
;; This protocol contains the function to manage the state
;; from a pipeline user perspective. The KeyValue store
;; can be backed by several different implementations
;; depending on the runtime configuration.
;;
(defprotocol KV
  "A key value store protocol"

  (set [kvstore sourceId key value]
    "Set the given key to the give value. Returns the new KV store")

  (get [kvstore sourceId key]
    "It returns the current value of the given key. nil if not found.")
;;  )


;;
;; The Transaction Log protocol is attached to a KeyValue store
;; to produce a event stream which can be restored when necessary
;; Also this protocol can have several different implementation
;; depending on the hosting cloud or the chosen solution.
;;
;;(defprotocol Tx-Log
;; "A transaction log protocol"

  (update [kvstore sourceId key value]
    "Set the given key to the give value. Returns the new KV store")

  (snapshot [kvstore]
    "Return the current value of the kvstore")

  (tx-log [kvstore]
    "Return the current Transaction log")

  (restore [kvstore tx-log]
    "restore the state of a KV store from a given txlog"))



(defrecord InMemoryKVstore [data]

  ;; implementing KV protocol
  KV
  (set [kvstore sourceId key value]
    (update kvstore sourceId key value))


  (get [kvstore sourceId key]
    (get-in kvstore [:data :snapshot sourceId key]))


  ;; implementing Tx-Log semantics
  ;; Tx-Log
  (update [kvstore sourceId key value]
    (InMemoryKVstore.
     (-> (:data kvstore)
         ;; increment version number
         (update-in [:version] inc)
         ;; update the given key
         (assoc-in [:snapshot sourceId key] value)
         ;; add a tx-log record
         ((fn [{:keys [version] :as data}]
            (update-in data [:tx-log] conj [version [sourceId key] value]))))))

  (tx-log [kvstore]
    (-> kvstore :data :tx-log))


  (snapshot [kvstore]
    (-> kvstore :data :snapshot))


  (restore [kvstore tx-log]<commit_msg>re-separating protocols<commit_after>(ns moebius.kv
  (:refer-clojure :exclude [set get remove update]))

;;
;; TODO: multi set function
;; TODO: force keys to string?
;; TODO: Do ve need the version? (think repartitioning)
;; TOOD: tx-log + compaction: single key or all keys x sourceId?
;;

;;
;; This protocol contains the function to manage the state
;; from a pipeline user perspective. The KeyValue store
;; can be backed by several different implementations
;; depending on the runtime configuration.
;;
(defprotocol KV
  "A key value store protocol"

  (set [kvstore sourceId key value]
    "Set the given key to the give value. Returns the new KV store")

  (get [kvstore sourceId key]
    "It returns the current value of the given key. nil if not found."))


;;
;; The Transaction Log protocol is attached to a KeyValue store
;; to produce a event stream which can be restored when necessary
;; Also this protocol can have several different implementation
;; depending on the hosting cloud or the chosen solution.
;;
(defprotocol Tx-Log
 "A transaction log protocol"

  (update [kvstore sourceId key value]
    "Set the given key to the give value. Returns the new KV store")

  (snapshot [kvstore]
    "Return the current value of the kvstore")

  (tx-log [kvstore]
    "Return the current Transaction log")

  (restore [kvstore tx-log]
    "restore the state of a KV store from a given txlog"))



(defrecord InMemoryKVstore [data]

  ;; implementing KV protocol
  KV
  (set [kvstore sourceId key value]
    (update kvstore sourceId key value))


  (get [kvstore sourceId key]
    (get-in kvstore [:data :snapshot sourceId key]))


  ;; implementing Tx-Log semantics
  Tx-Log
  (update [kvstore sourceId key value]
    (InMemoryKVstore.
     (-> (:data kvstore)
         ;; increment version number
         (update-in [:version] inc)
         ;; update the given key
         (assoc-in [:snapshot sourceId key] value)
         ;; add a tx-log record
         ((fn [{:keys [version] :as data}]
            (update-in data [:tx-log] conj [version [sourceId key] value]))))))

  (tx-log [kvstore]
    (-> kvstore :data :tx-log))


  (snapshot [kvstore]
    (-> kvstore :data :snapshot))


  (restore [kvstore tx-log]
{'fraction_non_alphanumeric': 0.10210951526032316, 'fraction_numerical': 0.0, 'mean_word_length': 3.3313896987366376, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6243917', 'n_tokens_mistral': 1346, 'n_tokens_neox': 1296, 'n_words': 564}