<commit_before>(ns co.nclk.linen.data
  (:require [clj-yaml.core :as yaml])
  (:gen-class))


(defprotocol PDataConnector
  (resolve-program [self m])
  (resolve-module [self m]))


(defn slurp-yaml
  [s]
  (let [s (if (.startsWith s (str "~" java.io.File/separator))
            (clojure.string/replace
              s #"~" (str (System/getProperty "user.home")))
            s)]
    (-> s slurp yaml/parse-string)))


(defrecord FileDataConnector []
  PDataConnector
  (resolve-program [self s]
    (slurp-yaml s))
  (resolve-module [self s]
    (slurp-yaml s)))
 

(defn connector [] (FileDataConnector.))

<commit_msg>parse-all instead of parse-string for co.nclk.linen.data<commit_after>(ns co.nclk.linen.data
  (:require [clj-yaml.core :as yaml])
  (:gen-class))


(defprotocol PDataConnector
  (resolve-program [self m])
  (resolve-module [self m]))


(defn slurp-yaml
  [s]
  (let [s (if (.startsWith s (str "~" java.io.File/separator))
            (clojure.string/replace
              s #"~" (str (System/getProperty "user.home")))
            s)]
    (-> s slurp yaml/parse-all)))


(defrecord FileDataConnector []
  PDataConnector
  (resolve-program [self s]
    (slurp-yaml s))
  (resolve-module [self s]
    (slurp-yaml s)))
 

(defn connector [] (FileDataConnector.))


{'fraction_non_alphanumeric': 0.16433021806853582, 'fraction_numerical': 0.0, 'mean_word_length': 2.929663608562691, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5418989', 'n_tokens_mistral': 490, 'n_tokens_neox': 472, 'n_words': 105}