<commit_before>(ns extracter.audit
  (:require [extracter.files :as files]
            [clojure.string :as s]
            [clojure.set :as set]
            [clojure.java.io :as io]))

(defn fact-names
  "Given a java.io.File object containing one or more documented facts, return the names of those facts."
  [^java.io.File f]
  (->> f
       slurp
       (re-seq #"[Ff]act: ([\w_<># ]*)")
       (mapcat rest)))

(defn likely-facts-in-dir
  "Recursively scans a directory for documented facts and returns the names
   of any facts found. Relies on the '# Fact: x' convention."
  [^String path]
  (->> path
       files/facts-in-dir
       (mapcat fact-names)
       set))

(defn find-missing
  "Given a path and a results set, print a list of facts that appear to be documented but are not in the results.
  Returns true if there are any missing facts, false otherwise."
  [^String path results]
  (let [actual (set (map :title results))
        expected (likely-facts-in-dir path)
        missing (set/difference expected actual)]
    (if (empty? missing)
      (do (println "Audit passed. All documented facts are present in the output.") false)
      (do (println "Missing from results:" (s/join ", " missing)) true))))<commit_msg>Make the auditing logic even more aggressive<commit_after>(ns extracter.audit
  (:require [extracter.files :as files]
            [clojure.string :as s]
            [clojure.set :as set]
            [clojure.java.io :as io]))

(defn fact-names
  "Given a java.io.File object containing one or more (possibly malformed) documented facts, return the
  names of those facts."
  [^java.io.File f]
  (->> f
       slurp
       (re-seq #"[Ff]act:[ ]+(.*)\n")
       (map second)))

(defn likely-facts-in-dir
  "Recursively scans a directory for documented facts and returns a set of the names of any facts found. 
  Relies on the 'Fact: x' convention, but applied much more loosely than the main parsing function."
  [^String path]
  (->> path
       io/file
       file-seq
       (filter files/ruby-file?)
       (mapcat fact-names)
       set))

(defn find-missing
  "Given a path and a results set, print a list of facts that appear to be documented but are not in the results.
  Returns true if there are any missing facts, false otherwise."
  [^String path results]
  (let [actual (set (map :title results))
        expected (likely-facts-in-dir path)
        missing (set/difference expected actual)]
    (if (empty? missing)
      (do (println "Audit passed. All documented facts are present in the output.") false)
      (do (println "Missing from results:" (s/join ", " missing)) true))))
{'fraction_non_alphanumeric': 0.11381804339550819, 'fraction_numerical': 0.0, 'mean_word_length': 2.8031837916063678, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '82904', 'n_tokens_mistral': 816, 'n_tokens_neox': 783, 'n_words': 317}