<commit_before>(ns tamaki.template.page
  (:require [clojure.java.io :as io]
            [clojure.string :as str]
            [markdown.core :as md]
            [clojure.string :as string]
            [clojure.set :as set]
            [net.cgrand.enlive-html :as ehtml]
            [tamaki-core.config :as config]
            [clojure-csv.core :as csv]
            [tamaki-core.file :as tfile])
  (:import (java.io StringWriter StringReader)
           (java.text SimpleDateFormat)))

(def ^:private date-formatter (SimpleDateFormat. "yyyy-MM-dd"))

(defn- load-md
  "Load a markdown file."
  ([md]
    (let [input (new StringReader (slurp md))
          output (new StringWriter)
          metadata (md/md-to-html input output :parse-meta? true :heading-anchors true)
          body (.toString output)]
      (merge {:body body :src md} {:metadata (merge metadata {:title (-> metadata :title first)
                                                      :template (-> metadata :template first)})}))))

(defn- read-page [md]
  (let [metadata (:metadata md)]
    (assoc md :metadata (assoc metadata :link (-> metadata :link first)))
    ))

(defn- write-page [page]
  (let [output (io/file (:dest (config/read-config)) (string/replace (-> page :metadata :link) #"^/" ""))
        template  (-> page :metadata :template)]
    (tfile/create-empty-file output)
    (require (symbol (str/replace  template #"/.*"  "")))
    (spit output ((var-get (resolve (symbol template))) page))))

(defn compile-mds
  ([page-root-dir]
   (doseq [md (-> page-root-dir io/file .listFiles)]
     (let [loaded (-> md .getAbsolutePath load-md)
           mod-loaded (read-page loaded)]
       (write-page mod-loaded))))
  ([] (compile-mds (:page-dir (config/read-config)))))


(defn- load-postmd [path]
  (letfn [(extract-date-from-filename [filename]
            (-> (re-matcher  #"^\d{4}-\d{1,2}-\d{1,2}" filename) re-find format-date))
          (format-date [date-str] (.parse date-formatter date-str))
          (build-link [filename] (string/replace filename
                                                 #"(\d{4})-(\d{1,2})-(\d{1,2})-(.+)\.(md|markdown)$"
                                                 "/$1/$2/$3/$4.html"))]
   (let [md (load-md path)
         link (build-link (-> path io/file .getName))
         output (str (:dest (config/read-config)) link)]
    (assoc md :link link :output output))))

(defn- create-neighbor-link [links]
  (map (fn [p n] {:previous-page p :next-page n})
       (cons nil (drop-last links))
       (concat (rest links) '(nil))))

(defn- append-neightbor-links [posts]
  (let [neighbor-links (create-neighbor-link (map #(:link %) posts))]
    (map (fn [m n] (merge m n)) posts neighbor-links)))

(defn load-post-excerpt [md]
  (let [compiled (load-postmd md)
        excerpt  (-> (ehtml/select (ehtml/html-resource (StringReader. (:body compiled))) [:p]) first ehtml/text)]
    (dissoc (assoc compiled :excerpt excerpt) :body)))

(defn load-recent-posts
  ([num post-dir]
   (let [mds (take num (-> (.listFiles (io/file post-dir)) reverse))]
     (map #(load-postmd (.getAbsolutePath %)) mds)))
  ([]
   (load-recent-posts (:recent-post-num (config/read-config)) (-> (config/read-config) :post-dir))))

(defn gen-paginate-page [dir]
  ^{:doc "dir is the directory where posts are put."}
  (letfn [(create-paginate-files [total paginate-url-pattern]
            (cons "index.html"
                  (map #(str/replace paginate-url-pattern #":num" (str %)) (range 2 (+ 1 total)))))]
    (let [num-per-page (:posts-per-page (config/read-config))
          excerpts (map #(load-post-excerpt (.getAbsolutePath %)) (-> dir io/file .listFiles reverse))
          parted-excerpts (partition-all num-per-page excerpts)
          output-files (create-paginate-files (count parted-excerpts) (:pagination-pattern (config/read-config)))
          neighbor-links (create-neighbor-link output-files)
          pages (map (fn [excerpt current-page neighbors] (merge {:posts excerpt} {:current-page current-page}  neighbors))
                      excerpts
                      output-files
                      neighbor-links)
          template (:pagination-template (config/read-config))]
      (require (symbol (str/replace template  #"/.*"  "")))
      (doseq [page pages]
        (let [dst (str (:dest (config/read-config)) "/" (:current-page page))]
          (tfile/create-empty-file dst)
          (spit dst ((var-get (resolve template)) page)))))))

(defn compile-postmds [dir]
  (let [posts (map #(-> % .getAbsolutePath load-postmd) (reverse (.listFiles (io/file dir))))
        posts-with-neighbors (append-neightbor-links posts)]
      (doseq [post posts-with-neighbors]
        (let [template (-> post :metadata :template)
              output (str (:dest (config/read-config)) (:link post))]
          (tfile/create-empty-file output)
          (require (symbol (str/replace template  #"/.*"  "")))
          (spit output ((var-get (resolve (symbol template))) post))))))
;

(defn load-markdown
  ([^String file]
   (letfn
     [(extract-date-from-filename [filename]
        (-> (re-matcher  #"^\d{4}-\d{1,2}-\d{1,2}" filename) re-find format-date))
      (format-date [date-str] (.parse date-formatter date-str))
      (build-dest [filename]<commit_msg>append  date atrribute to post map<commit_after>(ns tamaki.template.page
  (:require [clojure.java.io :as io]
            [clojure.string :as str]
            [markdown.core :as md]
            [clojure.string :as string]
            [clojure.set :as set]
            [net.cgrand.enlive-html :as ehtml]
            [tamaki-core.config :as config]
            [clojure-csv.core :as csv]
            [tamaki-core.file :as tfile])
  (:import (java.io StringWriter StringReader)
           (java.text SimpleDateFormat)))

(def ^:private date-formatter (SimpleDateFormat. "yyyy-MM-dd"))

(defn- load-md
  "Load a markdown file."
  ([md]
    (let [input (new StringReader (slurp md))
          output (new StringWriter)
          metadata (md/md-to-html input output :parse-meta? true :heading-anchors true)
          body (.toString output)]
      (merge {:body body :src md} {:metadata (merge metadata {:title (-> metadata :title first)
                                                      :template (-> metadata :template first)})}))))

(defn- read-page [md]
  (let [metadata (:metadata md)]
    (assoc md :metadata (assoc metadata :link (-> metadata :link first)))
    ))

(defn- write-page [page]
  (let [output (io/file (:dest (config/read-config)) (string/replace (-> page :metadata :link) #"^/" ""))
        template  (-> page :metadata :template)]
    (tfile/create-empty-file output)
    (require (symbol (str/replace  template #"/.*"  "")))
    (spit output ((var-get (resolve (symbol template))) page))))

(defn compile-mds
  ([page-root-dir]
   (doseq [md (-> page-root-dir io/file .listFiles)]
     (let [loaded (-> md .getAbsolutePath load-md)
           mod-loaded (read-page loaded)]
       (write-page mod-loaded))))
  ([] (compile-mds (:page-dir (config/read-config)))))


(defn- load-postmd [path]
  (letfn [(extract-date-from-filename [filename] (-> (re-seq #"^\d{4}-\d{1,2}-\d{1,2}" filename) first))
          (build-link [filename] (string/replace filename
                                                 #"(\d{4})-(\d{1,2})-(\d{1,2})-(.+)\.(md|markdown)$"
                                                 "/$1/$2/$3/$4.html"))]
   (let [md (load-md path)
         filename (-> path io/file .getName)
         link (build-link filename)
         output (str (:dest (config/read-config)) link)]
    (assoc md :link link :output output :date (extract-date-from-filename filename)))))

(defn- create-neighbor-link [links]
  (map (fn [p n] {:previous-page p :next-page n})
       (cons nil (drop-last links))
       (concat (rest links) '(nil))))

(defn- append-neightbor-links [posts]
  (let [neighbor-links (create-neighbor-link (map #(:link %) posts))]
    (map (fn [m n] (merge m n)) posts neighbor-links)))

(defn load-post-excerpt [md]
  (let [compiled (load-postmd md)
        excerpt  (-> (ehtml/select (ehtml/html-resource (StringReader. (:body compiled))) [:p]) first ehtml/text)]
    (dissoc (assoc compiled :excerpt excerpt) :body)))

(defn load-recent-posts
  ([num post-dir]
   (let [mds (take num (-> (.listFiles (io/file post-dir)) reverse))]
     (map #(load-postmd (.getAbsolutePath %)) mds)))
  ([]
   (load-recent-posts (:recent-post-num (config/read-config)) (-> (config/read-config) :post-dir))))

(defn gen-paginate-page [dir]
  ^{:doc "dir is the directory where posts are put."}
  (letfn [(create-paginate-files [total paginate-url-pattern]
            (cons "index.html"
                  (map #(str/replace paginate-url-pattern #":num" (str %)) (range 2 (+ 1 total)))))]
    (let [num-per-page (:posts-per-page (config/read-config))
          excerpts (map #(load-post-excerpt (.getAbsolutePath %)) (-> dir io/file .listFiles reverse))
          parted-excerpts (partition-all num-per-page excerpts)
          output-files (create-paginate-files (count parted-excerpts) (:pagination-pattern (config/read-config)))
          neighbor-links (create-neighbor-link output-files)
          pages (map (fn [excerpt current-page neighbors] (merge {:posts excerpt} {:current-page current-page}  neighbors))
                      parted-excerpts
                      output-files
                      neighbor-links)
          template (:pagination-template (config/read-config))]
      (require (symbol (str/replace template  #"/.*"  "")))
      (doseq [page pages]
        (let [dst (str (:dest (config/read-config)) "/" (:current-page page))]
          (tfile/create-empty-file dst)
          (spit dst ((var-get (resolve template)) page)))))))

(defn compile-postmds [dir]
  (let [posts (map #(-> % .getAbsolutePath load-postmd) (reverse (.listFiles (io/file dir))))
        posts-with-neighbors (append-neightbor-links posts)]
      (doseq [post posts-with-neighbors]
        (let [template (-> post :metadata :template)
              output (str (:dest (config/read-config)) (:link post))]
          (tfile/create-empty-file output)
          (require (symbol (str/replace template  #"/.*"  "")))
          (spit output ((var-get (resolve (symbol template))) post))))))
;

(defn load-markdown
  ([^String file]
   (letfn
     [(extract-date-from-filename [filename]
        (-> (re-matcher  #"^\d{4}-\d{1,2}-\d{1,2}" filename) re-find format-date))
      (format-date [date-str] (.parse date-formatter date-str))
      (build-dest [filename]
{'fraction_non_alphanumeric': 0.17414347908385386, 'fraction_numerical': 0.003975014196479273, 'mean_word_length': 2.493223140495868, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3075337', 'n_tokens_mistral': 3534, 'n_tokens_neox': 3371, 'n_words': 853}