<commit_before>(ns powderkeg.core
  "A spark lib."
  (:refer-clojure :exclude [into shuffle])
  (:require 
    [powderkeg.ouroboros :as ou] ; must be first
    [powderkeg.kryo :as kryo]
    [powderkeg.pool :as pool]
    [cemerick.pomegranate.aether :as mvn]
    [clojure.core :as clj]
    [clojure.java.io :as io]
    [clojure.string :as str]
    [clojure.core.protocols :refer [coll-reduce CollReduce]]
    [clojure.core.reducers :as r]
    [net.cgrand.xforms :as x]))

(defn- sig [x]
  (cond
    (instance? java.lang.reflect.Method x) 
    [(.getReturnType x) (.getName x) (vec (.getParameterTypes x))]
    (instance? java.lang.reflect.Constructor x)
    [nil "<init>" (vec (.getParameterTypes x))]))

(defn- has-class? [^Class class msig]
  (some #(= msig (sig %)) (concat (.getMethods class) (.getConstructors class))))

(defmacro ^:private compile-cond [& choices]
	(let [x (Object.)
	      expr
	      (reduce (fn [_ [test expr]]
                  (when (eval test) (reduced expr)))
         x (partition 2 choices))]
	  (when (= x expr)
	    (throw (ex-info "No valid choice." {:form &form})))))

(defn- all-files
  "Returns a map of relative paths (as Strings) to Files for all files (not directories) below the argument."
  [^java.io.File f]
  (let [f (.getCanonicalFile f)
        path (.getCanonicalPath f)
        dir? #(.isDirectory ^java.io.File %)
        dot? #(.startsWith (.getName ^java.io.File %) ".")]
    (clj/into {}
      (comp
        (remove (some-fn dir? dot?))
        (keep #(let [p (.getCanonicalPath ^java.io.File %)]
                 ; the test below is for links, I didn't check how they behave with list and canonpath
                 (when (.startsWith p path) [(subs p (inc (count path))) %]))))
      (tree-seq (every-pred dir? (complement dot?)) #(.listFiles ^java.io.File %) f))))

(defn package-env!<commit_msg>this is what happens when you insert a last minute assertion...<commit_after>(ns powderkeg.core
  "A spark lib."
  (:refer-clojure :exclude [into shuffle])
  (:require 
    [powderkeg.ouroboros :as ou] ; must be first
    [powderkeg.kryo :as kryo]
    [powderkeg.pool :as pool]
    [cemerick.pomegranate.aether :as mvn]
    [clojure.core :as clj]
    [clojure.java.io :as io]
    [clojure.string :as str]
    [clojure.core.protocols :refer [coll-reduce CollReduce]]
    [clojure.core.reducers :as r]
    [net.cgrand.xforms :as x]))

(defn- sig [x]
  (cond
    (instance? java.lang.reflect.Method x) 
    [(.getReturnType x) (.getName x) (vec (.getParameterTypes x))]
    (instance? java.lang.reflect.Constructor x)
    [nil "<init>" (vec (.getParameterTypes x))]))

(defn- has-class? [^Class class msig]
  (some #(= msig (sig %)) (concat (.getMethods class) (.getConstructors class))))

(defmacro ^:private compile-cond [& choices]
  (let [x (Object.)
        expr
        (reduce (fn [_ [test expr]]
                  (when (eval test) (reduced expr)))
          x (partition 2 choices))]
    (when (= x expr)
      (throw (ex-info "No valid choice." {:form &form})))
    expr))

(defn- all-files
  "Returns a map of relative paths (as Strings) to Files for all files (not directories) below the argument."
  [^java.io.File f]
  (let [f (.getCanonicalFile f)
        path (.getCanonicalPath f)
        dir? #(.isDirectory ^java.io.File %)
        dot? #(.startsWith (.getName ^java.io.File %) ".")]
    (clj/into {}
      (comp
        (remove (some-fn dir? dot?))
        (keep #(let [p (.getCanonicalPath ^java.io.File %)]
                 ; the test below is for links, I didn't check how they behave with list and canonpath
                 (when (.startsWith p path) [(subs p (inc (count path))) %]))))
      (tree-seq (every-pred dir? (complement dot?)) #(.listFiles ^java.io.File %) f))))

(defn package-env!
{'fraction_non_alphanumeric': 0.1600743297053358, 'fraction_numerical': 0.0005309264666843643, 'mean_word_length': 2.756729810568295, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2538321', 'n_tokens_mistral': 1379, 'n_tokens_neox': 1303, 'n_words': 394}