<commit_before>(ns fleetdb.embedded
  (:use [fleetdb.util :only (def- ? spawn rassert raise)]
        [clojure.contrib.seq-utils :only (partition-all)])
  (:require (fleetdb [core :as core] [fair-lock :as fair-lock]
                     [io :as io] [file :as file] [lint :as lint]))
  (:import  (java.util ArrayList)))

(defn- dba? [dba]
  (? (:write-lock (meta dba))))

(defn- persistent? [dba]
  (? (:write-dos (meta dba))))

(defn- ephemeral? [dba]
  (not (persistent? dba)))

(defn- compacting? [dba]
  (? (:write-buf (meta dba))))

(defn- replay-query [db q]
  (first (core/query db q)))

(defn- read-from [read-path]
  (let [dis      (io/dis-init read-path)
        queries  (io/dis-deserialized-seq dis)
        empty    (core/init)]
    (reduce replay-query empty queries)))

(defn- write-to [db write-path]
  (let [dos (io/dos-init write-path)]
    (doseq [coll (core/query db [:list-collections])]
      (doseq [records (partition-all 100 (core/query db [:select coll]))]
        (io/dos-serialize dos [:insert coll (vec records)]))
      (doseq [ispec (core/query db [:list-indexes coll])]
        (io/dos-serialize dos [:create-index coll ispec])))
    (io/dos-close dos)))

(defn- init* [db & [other-meta]]
  (atom db :meta (assoc other-meta :write-lock (fair-lock/init))))

(defn init-ephemeral []
  (init* (core/init)))

(defn load-ephemeral [read-path]
  (init* (read-from read-path)))

(defn init-persistent [write-path]
  (let [write-dos (io/dos-init write-path)]
    (init* (core/init) {:write-dos write-dos :write-path write-path})))

(defn load-persistent [read-write-path]
  (let [db        (read-from read-write-path)
        write-dos (io/dos-init read-write-path)]
    (init* db {:write-dos write-dos :write-path read-write-path})))

(defn close [dba]
  (assert (fair-lock/join (:write-lock (meta dba)) 60))
  (if-let [write-dos (:write-dos (meta dba))]
    (io/dos-close write-dos))
  (assert (compare-and-set! dba @dba nil))
  true)

(defn fork [dba]
  (rassert (ephemeral? dba) "cannot fork persistent databases")<commit_msg>Sketch updated embedded module.<commit_after>(ns fleetdb.embedded
  (:use [fleetdb.util :only (def- ? spawn rassert raise)]
        [clojure.contrib.seq-utils :only (partition-all)])
  (:require (fleetdb [core :as core] [fair-lock :as fair-lock]
                     [io :as io] [file :as file] [lint :as lint]))
  (:import  (java.util ArrayList)))

(defn- dba? [dba]
  (? (:write-lock (meta dba))))

(defn- persistent? [dba]
  (? (:write-dos (meta dba))))

(defn- ephemeral? [dba]
  (not (persistent? dba)))

(defn- compacting? [dba]
  (? (:write-buf (meta dba))))

(defn- replay-query [db q]
  (first (core/query db q)))

(defn- read-from [read-path]
  (let [dis      (io/dis-init read-path)
        queries  (io/dis-deserialized-seq dis)
        empty    (core/init)]
    (reduce replay-query empty queries)))

(defn- write-to [db write-path]
  (let [dos (io/dos-init write-path)]
    (doseq [coll (core/query db ["list-collections"])]
      (doseq [records (partition-all 100 (core/query db ["select" coll]))]
        (io/dos-serialize dos ["insert" coll (vec records)]))
      (doseq [ispec (core/query db ["list-indexes" coll])]
        (io/dos-serialize dos ["create-index" coll ispec])))
    (io/dos-close dos)))

(defn- init* [db & [other-meta]]
  (atom db :meta (assoc other-meta :write-lock (fair-lock/init))))

(defn init-ephemeral []
  (init* (core/init)))

(defn load-ephemeral [read-path]
  (init* (read-from read-path)))

(defn init-persistent [write-path]
  (let [write-dos (io/dos-init write-path)]
    (init* (core/init) {:write-dos write-dos :write-path write-path})))

(defn load-persistent [read-write-path]
  (let [db        (read-from read-write-path)
        write-dos (io/dos-init read-write-path)]
    (init* db {:write-dos write-dos :write-path read-write-path})))

(defn close [dba]
  (assert (fair-lock/join (:write-lock (meta dba)) 60))
  (if-let [write-dos (:write-dos (meta dba))]
    (io/dos-close write-dos))
  (assert (compare-and-set! dba @dba nil))
  true)

(defn fork [dba]
  (rassert (ephemeral? dba) "cannot fork persistent databases")
{'fraction_non_alphanumeric': 0.18648583192056187, 'fraction_numerical': 0.002421893921046258, 'mean_word_length': 3.253347064881565, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2825071', 'n_tokens_mistral': 1666, 'n_tokens_neox': 1559, 'n_words': 408}