<commit_before>(ns incise.parsers.markdown
  (:require [incise.parsers.core :as pc]))

(defn parse
  "Parse markdown file returning a Parse record."
  [^java.io.File file]
  (do (println (.getPath file))
      ^pc/Parse (pc/map->Parse {:title "Yoyoyo"
                                :layout :page
                                :content "yoyoyo"})))

(pc/register [:md :markdown] parse)
<commit_msg>Fill in markdown parser more.<commit_after>(ns incise.parsers.markdown
  (:require [incise.parsers.core :as pc]
            [incise.parsers.helpers :as ph]
            [incise.config :refer [config]]
            [markdown.core :as md])
  (:import [java.io File StringWriter]))

(defn md-file->string
  "Parse the given file as markdown returning html."
  [^File file]
  (let [output (StringWriter.)]
    (md/md-to-html-string file output)
    (str output)))

(defrecord Markdown [file]
  pc/Inciseable
  (parse [this]
    (pc/map->Parse {:content (md-file->string (:file this))}))
  (incise [this parse layout]
    (ph/Parse->path parse)
    (layout @config (:content parse))))

(pc/register [:md :markdown] ->Markdown)

{'fraction_non_alphanumeric': 0.14986619090098127, 'fraction_numerical': 0.0, 'mean_word_length': 2.7905405405405403, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5263521', 'n_tokens_mistral': 382, 'n_tokens_neox': 363, 'n_words': 101}