<commit_before>(ns fat-jar.core
  (:require [clojure.tools.cli :refer [parse-opts]]
                 [clojure.java.io :as io]
                 [clojure.tools.reader.edn :as edn]
                 [clj-json.core :as json]
                 [clojurewerkz.propertied.properties :as p])
  (:gen-class))
(import '(java.io StringReader BufferedReader))

; guide for cli-options: https://github.com/clojure/tools.cli/blob/master/src/test/clojure/clojure/tools/cli_test.clj
(def cli-options
  [["-o" "--output DIRECTORY" "Folder where to write the file"
     :missing "output option is missing"]
   ["-t" "--type TYPE" "type of input"
     :missing "type option is missing"]

   ["-h" "--help"]])

(defn select- [data-mapping coll]
  """Navigate using the selector"""
  (let [selector (:map data-mapping)
         output (:to data-mapping)]
    {:output output
      :v(reduce #(% %2) coll selector)}))

(defn write-file [filename payload]
   (with-open [w (clojure.java.io/writer filename)]
  (doseq [line payload]
    (.write w line)
    (.newLine w))))

(let [pl (p/load-from {"key" "value"})]
  (p/properties->map pl true))

(def config
  (p/properties->map 
    (->> "./mapper.properties" io/file p/load-from)
    true))
<commit_msg>defect<commit_after>(ns fat-jar.core
  (:require [clojure.tools.cli :refer [parse-opts]]
                 [clojure.java.io :as io]
                 [clojure.tools.reader.edn :as edn]
                 [clj-json.core :as json]
                 [clojurewerkz.propertied.properties :as p])
  (:gen-class))
(import '(java.io StringReader BufferedReader))

; guide for cli-options: https://github.com/clojure/tools.cli/blob/master/src/test/clojure/clojure/tools/cli_test.clj
(def cli-options
  [["-o" "--output DIRECTORY" "Folder where to write the file"
     :missing "output option is missing"]
   ["-t" "--type TYPE" "type of input"
     :missing "type option is missing"]

   ["-h" "--help"]])

(defn select- [data-mapping coll]
  """Navigate using the selector"""
  (let [selector (:map data-mapping)
         output (:to data-mapping)]
    {:output output
      :v (reduce #(% %2) coll selector)}))

(defn write-file [filename payload]
   (with-open [w (clojure.java.io/writer filename)]
  (doseq [line payload]
    (.write w line)
    (.newLine w))))

(def config
  (p/properties->map 
    (->> "./mapper.properties" io/file p/load-from)
    true))

{'fraction_non_alphanumeric': 0.16778805719091675, 'fraction_numerical': 0.0008410428931875525, 'mean_word_length': 2.7702060221870046, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 10, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '618539', 'n_tokens_mistral': 841, 'n_tokens_neox': 823, 'n_words': 209}