<commit_before>(ns incise.parsers.helpers
  (:require (clj-time [core :as tm]
                      [coerce :as tc])
            [clojure.string :as s])
  (:import [java.io File]))

(defn dashify
  "Dashify a title, replacing all non word characters with a dash."
  [^String title]
  (-> title
      (s/lower-case)
      (s/replace #"[^\w]" "-")))

(defn date-time->path
  "Convert a DateTime to a path like string."
  [date-time]
  (str (when date-time
         (str (tm/year date-time) \/
              (tm/month date-time) \/
              (tm/day date-time) \/))))

(def date-str->path (comp date-time->path tc/from-string))

(defn Parse->path
  [^Parse {:keys [path layout date title extension]}]
  (or path
      (str (when (= layout :post) (date-str->path date))
           (dashify title)
           extension)))

(defn extension [^File file]
  "Get the extension, enforcing lower case, on the given file."
  (-> file
      (.getName)
      (s/split #"\.")
      (last)
      (s/lower-case)))<commit_msg>Ignore post key in Parse->path.<commit_after>(ns incise.parsers.helpers
  (:require (clj-time [core :as tm]
                      [coerce :as tc])
            [clojure.string :as s])
  (:import [java.io File]))

(defn dashify
  "Dashify a title, replacing all non word characters with a dash."
  [^String title]
  (-> title
      (s/lower-case)
      (s/replace #"[^\w]" "-")))

(defn date-time->path
  "Convert a DateTime to a path like string."
  [date-time]
  (str (when date-time
         (str (tm/year date-time) \/
              (tm/month date-time) \/
              (tm/day date-time) \/))))

(def date-str->path (comp date-time->path tc/from-string))

(defn Parse->path
  [^Parse {:keys [path date title extension]}]
  (or path
      (str (date-str->path date)
           (dashify title)
           extension)))

(defn extension [^File file]
  "Get the extension, enforcing lower case, on the given file."
  (-> file
      (.getName)
      (s/split #"\.")
      (last)
      (s/lower-case)))
{'fraction_non_alphanumeric': 0.16459472899055197, 'fraction_numerical': 0.0, 'mean_word_length': 2.1635220125786163, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1531309', 'n_tokens_mistral': 698, 'n_tokens_neox': 678, 'n_words': 199}