<commit_before>(ns doctopus.storage.impls.temp-fs
  (:require [clojure.string :as str]
            [doctopus.files :as files]
            [doctopus.storage.impls.fs-impl :refer [save-html-file] :as fs-impl]
            [me.raynes.fs :as fs]))

;; The root of the temp filesystem. Each Thing will store its stuff
;; within this directory
(def temp-dir (atom (fs/temp-dir  "doctopus-temp")))

(defn regenerate-temp-dir
  "Generates a new temp dir"
  []
  (swap! temp-dir fs/temp-dir "doctopus-temp"))

(defn save-fn
  [key path-html-pairs]
  (doseq [[path html] path-html-pairs] (save-html-file @temp-dir key path html)))

(defn load-fn
  "This is as Version 1 a way of doing this as I can think of:

  1. We load html
  2. We wrap that html in a function that returns itself, so we can Bidi"
  [key]
  (binding [fs/*cwd* @temp-dir]
    (let [dir (fs/file key)
          rel-path-html-pairs (files/read-html dir)]
      (into [] (map (fn [[rel-path html]] [rel-path (fn [_] html)]) rel-path-html-pairs)))))

(defn remove-fn
  [key]
  (fs-impl/remove-html @temp-dir key))
<commit_msg>Writing the tentacle test revealed the temp-fs implementation to be fucked. This is a comparatively quick-and-dirty fix; these two implementations share a *ton* of code, and should be compressed/refactored<commit_after>(ns doctopus.storage.impls.temp-fs
  (:require [clojure.string :as str]
            [doctopus.files :as files]
            [doctopus.storage.impls.fs-impl :refer [save-html-file] :as fs-impl]
            [me.raynes.fs :as fs]
            [bidi.ring :as bidi-ring]))

;; The root of the temp filesystem. Each Thing will store its stuff
;; within this directory
(def temp-dir (atom (fs/temp-dir  "doctopus-temp")))

(defn regenerate-temp-dir
  "Generates a new temp dir"
  []
  (swap! temp-dir fs/temp-dir "doctopus-temp"))

(defn save-fn
  "Save a dir of html stuff by moving it in to the current temp root"
  [key src-dir]
  (binding [fs/*cwd* @temp-dir]
    (let [target-dir (fs/file key)
          result (fs/copy-dir src-dir target-dir)]
      (and (fs/exists? result)
           (fs/readable? result)))))

(defn load-fn
  "Returns the routes this serves"
  [key]
  (let [file-handle (binding [fs/*cwd* @temp-dir] (fs/file key))
        file-name (str (.getPath file-handle) "/")]
    (if (fs/exists? file-handle)
      [key {"/" (bidi-ring/->Files {:dir file-name})}]
      nil)))

(defn remove-fn
  [key]
  (fs-impl/remove-html @temp-dir key))

{'fraction_non_alphanumeric': 0.14659044507962435, 'fraction_numerical': 0.0012249897917517355, 'mean_word_length': 2.926282051282051, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1198890', 'n_tokens_mistral': 896, 'n_tokens_neox': 868, 'n_words': 270}