<commit_before>(ns taoensso.carmine.tundra
  "Semi-automatic archival datastore layer for Carmine.
  Use multiple Redis instances (recommended) or Redis databases for local key
  namespacing.

  Redis keys:
    * carmine:tundra:evictable - set, ks for which `ensure-ks` fail should throw"

  {:author "<NAME> (@ptaoussanis)"}
  (:require [taoensso.encore      :as enc]
            [taoensso.nippy       :as nippy]
            [taoensso.nippy.tools :as nippy-tools]
            [taoensso.timbre      :as timbre]
            [taoensso.carmine     :as car :refer (wcar)]
            [taoensso.carmine.message-queue :as mq])
  (:import  [java.net URLDecoder URLEncoder]))

;;;; TODO v2/refactor
;; - General refactor (reduce-based flow, refactor stores, etc.)
;; - Drop mq for a simpler zset {<k> <udt-last-dirty>} based model:
;;   atomic-pull ks on verified write when udt unchanged against write val

;;;; Public interfaces

(defprotocol IDataStore "Extension point for additional datastores."

  ;; Done 1-at-a-time via mq, so no need for bulk ks api:
  (put-key    [dstore k v] "(put-key dstore \"key\" <frozen-val>) => <#{true <ex>}>")
  (fetch-keys [dstore ks] "(fetch-keys dstore [\"key\" ...]) => [<#{<frozen-val> <ex>}> ...]"))

(defprotocol IFreezer "Extension point for compressors, encryptors, etc."
  (freeze [freezer x] "Returns datastore-ready key val.
                       (comp put-key freeze): dump-ba -> datastore val.")
  (thaw   [freezer x] "Returns Redis-ready key val.
                       (comp thaw fetch-key): datastore val -> dump-ba."))

(defprotocol ITundraStore
  (ensure-ks* [tstore ks])
  (dirty*     [tstore ks])
  (worker     [tstore conn-opts wopts]
    "Alpha - subject to change.
    Returns a threaded message queue worker to routinely freeze Redis keys
    marked as dirty to datastore and mark successfully frozen keys as clean.
    Logs any errors. THESE ERRORS ARE **IMPORTANT**: an email or other<commit_msg>Housekeeping<commit_after>(ns taoensso.carmine.tundra
  "Semi-automatic archival datastore layer for Carmine.
  Use multiple Redis instances (recommended) or Redis databases for local key
  namespacing.

  Redis keys:
    * carmine:tundra:evictable - set, ks for which `ensure-ks` fail should throw"

  {:author "<NAME> (@ptaoussanis)"}
  (:require [taoensso.encore      :as enc]
            [taoensso.nippy       :as nippy]
            [taoensso.nippy.tools :as nippy-tools]
            [taoensso.timbre      :as timbre]
            [taoensso.carmine     :as car :refer (wcar)]
            [taoensso.carmine.message-queue :as mq])
  (:import  [java.net URLDecoder URLEncoder]))

;;;; TODO v2/refactor
;; - Why is evictable set key not configurable?
;; - General refactor (reduce-based flow, refactor stores, etc.)
;; - Drop mq for a simpler zset {<k> <udt-last-dirty>} based model:
;;   atomic-pull ks on verified write when udt unchanged against write val

;;;; Public interfaces

(defprotocol IDataStore "Extension point for additional datastores."

  ;; Done 1-at-a-time via mq, so no need for bulk ks api:
  (put-key    [dstore k v] "(put-key dstore \"key\" <frozen-val>) => <#{true <ex>}>")
  (fetch-keys [dstore ks] "(fetch-keys dstore [\"key\" ...]) => [<#{<frozen-val> <ex>}> ...]"))

(defprotocol IFreezer "Extension point for compressors, encryptors, etc."
  (freeze [freezer x] "Returns datastore-ready key val.
                       (comp put-key freeze): dump-ba -> datastore val.")
  (thaw   [freezer x] "Returns Redis-ready key val.
                       (comp thaw fetch-key): datastore val -> dump-ba."))

(defprotocol ITundraStore
  (ensure-ks* [tstore ks])
  (dirty*     [tstore ks])
  (worker     [tstore conn-opts wopts]
    "Alpha - subject to change.
    Returns a threaded message queue worker to routinely freeze Redis keys
    marked as dirty to datastore and mark successfully frozen keys as clean.
    Logs any errors. THESE ERRORS ARE **IMPORTANT**: an email or other
{'fraction_non_alphanumeric': 0.12503163756011137, 'fraction_numerical': 0.0010124019235636548, 'mean_word_length': 2.971859296482412, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7126943', 'n_tokens_mistral': 1351, 'n_tokens_neox': 1299, 'n_words': 428}