<commit_before>(ns vault.system
  (:require
    (clj-time
      [core :as time]
      [format :as ftime])
    [clojure.java.io :as io]
    [clojure.repl :refer :all]
    [clojure.string :as str]
    [clojure.tools.namespace.repl :refer [refresh]]
    [environ.core :refer [env]]
    (puget
      [data]
      [printer :as puget])
    [vault.blob.core :as blob]
    vault.blob.store
    [vault.data.core :as data]
    [vault.entity.core :as entity]
    [vault.index.core :as index]
    (vault.tool
      [config :as config])))


;; GENERAL CONFIG

(puget.data/extend-tagged-map vault.blob.store.Blob 'vault.tool/blob)



;; VAULT SYSTEM

(def config
  (config/load-configs (env :vault-config "dev/config")))


(def blobs
  (config/select-blob-store
    (:blob-stores config)
    :default))<commit_msg>Render datoms as tagged vectors in tool.<commit_after>(ns vault.system
  (:require
    (clj-time
      [core :as time]
      [format :as ftime])
    [clojure.java.io :as io]
    [clojure.repl :refer :all]
    [clojure.string :as str]
    [clojure.tools.namespace.repl :refer [refresh]]
    [environ.core :refer [env]]
    (puget
      [data]
      [printer :as puget])
    [vault.blob.core :as blob]
    vault.blob.store
    [vault.data.core :as data]
    [vault.entity.core :as entity]
    [vault.index.core :as index]
    (vault.tool
      [config :as config])))


;; GENERAL CONFIG

(puget.data/extend-tagged-map vault.blob.store.Blob 'vault.tool/blob)
(puget.data/extend-tagged-value
  vault.entity.datom.Datom 'vault.tool/datom
  (juxt :op :entity :attribute :value :tx :time))



;; VAULT SYSTEM

(def config
  (config/load-configs (env :vault-config "dev/config")))


(def blobs
  (config/select-blob-store
    (:blob-stores config)
    :default))
{'fraction_non_alphanumeric': 0.15108323831242873, 'fraction_numerical': 0.0, 'mean_word_length': 2.891352549889135, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4854398', 'n_tokens_mistral': 679, 'n_tokens_neox': 669, 'n_words': 147}