<commit_before>(ns lambda-blog.parsers.md
  (:require [clojure.stacktrace :refer [print-stack-trace]]
            [markdown.core :refer [md-to-html-string md-to-html-string-with-meta]]
            [taoensso.timbre :as log]))

(defn- parse-metadata [metadata]
  (->> metadata
       seq
       (map (fn [[k v]]
              [k (if (= (count v) 1)
                   (first v)
                   v)]))
       (into {})))

(defn- do-parse [contents]
  (try (md-to-html-string-with-meta contents)
       (catch java.lang.NullPointerException e
         (log/warnf "Caught an exception while parsing input file (bad metadata format?): %s" e)
         (log/debug (with-out-str (print-stack-trace e)))
         {:metadata nil
          :html (md-to-html-string contents)})))

(defn parse [contents]
  (if-not (empty? contents)
    (let [{:keys [metadata html]} (do-parse contents)]
      {:metadata (parse-metadata metadata)
       :contents html})
    {:metadata {}
     :contents ""}))<commit_msg>Added docstrings to lambda-blog.parsers.md namespace.<commit_after>(ns lambda-blog.parsers.md
  "Markdown parser."
  (:require [clojure.stacktrace :refer [print-stack-trace]]
            [markdown.core :refer [md-to-html-string md-to-html-string-with-meta]]
            [taoensso.timbre :as log]))

(defn- parse-metadata [metadata]
  (->> metadata
       seq
       (map (fn [[k v]]
              [k (if (= (count v) 1)
                   (first v)
                   v)]))
       (into {})))

(defn- do-parse [contents]
  (try (md-to-html-string-with-meta contents)
       (catch java.lang.NullPointerException e
         (log/warnf "Caught an exception while parsing input file (bad metadata format?): %s" e)
         (log/debug (with-out-str (print-stack-trace e)))
         {:metadata nil
          :html (md-to-html-string contents)})))

(defn parse
  "Parses file `contents` as a Markdown document and returns HTML and various bits of metadata."
  [contents]
  (if-not (empty? contents)
    (let [{:keys [metadata html]} (do-parse contents)]
      {:metadata (parse-metadata metadata)
       :contents html})
    {:metadata {}
     :contents ""}))
{'fraction_non_alphanumeric': 0.15191053122087605, 'fraction_numerical': 0.0009319664492078285, 'mean_word_length': 2.3546875, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3603822', 'n_tokens_mistral': 691, 'n_tokens_neox': 682, 'n_words': 183}