<commit_before>(ns blocks.store.file-test
  (:require
    (blocks.store
      [file :as file :refer [file-block-store]]
      [tests :as tests])
    [clojure.java.io :as io]
    [clojure.test :refer :all]))


(deftest conversion-tests
  (testing "file->id"
    (let [file->id @#'file/file->id]
      (is (nil? (file->id nil nil))
          "should return nil for nil files")
      (is (nil? (file->id "foo/bar" (io/file "foo" "baz" "abc")))
          "should return nil for non-child files")
      (is (nil? (file->id "foo/bar" (io/file "foo" "bar" "1220" "ab" "0x" "123abc")))
          "should return nil for non-hex files"))))


(deftest ^:integration test-file-store
  (let [tmpdir (io/file "target" "test" "tmp"
                        (str "file-block-store."
                          (System/currentTimeMillis)))]
    (tests/check-store!
      #(file-block-store tmpdir)
      :eraser file/erase!)))
<commit_msg>Add test exercising file store prefix listing.<commit_after>(ns blocks.store.file-test
  (:require
    [blocks.core :as block]
    (blocks.store
      [file :as file :refer [file-block-store]]
      [tests :as tests])
    [clojure.java.io :as io]
    [clojure.test :refer :all]))


(defn- mk-tmpdir!
  []
  (io/file "target" "test" "tmp"
           (str "file-block-store." (System/currentTimeMillis))))


(deftest conversion-tests
  (testing "file->id"
    (let [file->id @#'file/file->id]
      (is (nil? (file->id nil nil))
          "should return nil for nil files")
      (is (nil? (file->id "foo/bar" (io/file "foo" "baz" "abc")))
          "should return nil for non-child files")
      (is (nil? (file->id "foo/bar" (io/file "foo" "bar" "1220" "ab" "0x" "123abc")))
          "should return nil for non-hex files"))))


(deftest file-listing
  (let [store (file-block-store (mk-tmpdir!))
        a (block/read! "larry")
        b (block/read! "curly")
        c (block/read! "moe")]
    (block/put! store a)
    (block/put! store b)
    (block/put! store c)
    (is (= 3 (count (block/list store :after "12200c"))))
    (is (= 3 (count (block/list store :after "12200d0980"))))
    (is (= 1 (count (block/list store :after "12204b6f51"))))
    (is (= (:id b) (:id (first (block/list store :after "12204b6f51")))))
    (is (empty? (block/list store :after "122064")))))


(deftest ^:integration test-file-store
  (let [tmpdir (mk-tmpdir!)]
    (tests/check-store!
      #(file-block-store tmpdir)
      :eraser file/erase!)))

{'fraction_non_alphanumeric': 0.18581907090464547, 'fraction_numerical': 0.02241238793806031, 'mean_word_length': 2.6587183308494784, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '340752', 'n_tokens_mistral': 940, 'n_tokens_neox': 894, 'n_words': 240}