<commit_before>(ns jkpl.parse
  (:require [clojure.string :as string]
            [clojure.edn :as edn]
            [jkpl.utils :as utils]
            [clj-time.core :as t]
            [clj-time.format :as tf]))

(def page-separator #"\n\n---\n\n")

(def date-formatter (tf/formatter "yyyy-MM-dd"))

(def date-pattern #"\d+-\d+-\d+")

(defn- parse-date
  [s]
  (tf/parse date-formatter (re-find date-pattern s)))

(defn- posts->tags
  [posts]
  (->> posts (mapcat :tags) distinct sort))

(defn- link-post
  ([prev current next]
    (assoc current :prev prev :next next))
  ([prev current]
   (link-post prev current nil)))

(defn- link-posts
  [posts]
  (let [first-post (link-post nil (first posts) (second posts))
        linked-posts (map (partial apply link-post) (partition 3 1 [] posts))]
    (cons first-post linked-posts)))

(defn- latest-posts
  [conf posts]
  (take (get conf :latest-posts 5) posts))

(defn- parse-content
  [content]
  (let [parts (string/split content page-separator 2)
        page-meta (edn/read-string (first parts))
        content (second parts)]
    (assoc page-meta :content content)))

(defn- parse-page
  [conf filename content & {:keys [page-type] :or {page-type ""}}]
  (assoc (parse-content content)
         :uri (utils/uri page-type filename)))

(defn- parse-post
  [conf filename content]
  (assoc (parse-page conf filename content :page-type "post")
         :date (parse-date filename)))

(defn- parse-pages
  [conf coll]
  (->> coll
       (map (partial apply parse-page conf))
       (sort-by :title)))

(defn- parse-posts
  [conf coll]
  (->> coll
       (map (partial apply parse-post conf))
       (sort-by :date)<commit_msg>Reverse date sorting for posts<commit_after>(ns jkpl.parse
  (:require [clojure.string :as string]
            [clojure.edn :as edn]
            [jkpl.utils :as utils]
            [clj-time.core :as t]
            [clj-time.format :as tf]))

(def page-separator #"\n\n---\n\n")

(def date-formatter (tf/formatter "yyyy-MM-dd"))

(def date-pattern #"\d+-\d+-\d+")

(defn- parse-date
  [s]
  (tf/parse date-formatter (re-find date-pattern s)))

(defn- posts->tags
  [posts]
  (->> posts (mapcat :tags) distinct sort))

(defn- link-post
  ([prev current next]
    (assoc current :prev prev :next next))
  ([prev current]
   (link-post prev current nil)))

(defn- link-posts
  [posts]
  (let [first-post (link-post nil (first posts) (second posts))
        linked-posts (map (partial apply link-post) (partition 3 1 [] posts))]
    (cons first-post linked-posts)))

(defn- latest-posts
  [conf posts]
  (take (get conf :latest-posts 5) posts))

(defn- parse-content
  [content]
  (let [parts (string/split content page-separator 2)
        page-meta (edn/read-string (first parts))
        content (second parts)]
    (assoc page-meta :content content)))

(defn- parse-page
  [conf filename content & {:keys [page-type] :or {page-type ""}}]
  (assoc (parse-content content)
         :uri (utils/uri page-type filename)))

(defn- parse-post
  [conf filename content]
  (assoc (parse-page conf filename content :page-type "post")
         :date (parse-date filename)))

(defn- reverse-compare [a b] (compare b a))

(defn- parse-pages
  [conf coll]
  (->> coll
       (map (partial apply parse-page conf))
       (sort-by :title)))

(defn- parse-posts
  [conf coll]
  (->> coll
       (map (partial apply parse-post conf))
       (sort-by :date reverse-compare)
{'fraction_non_alphanumeric': 0.1618075801749271, 'fraction_numerical': 0.0023323615160349854, 'mean_word_length': 2.8164627363737487, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '722756', 'n_tokens_mistral': 1231, 'n_tokens_neox': 1201, 'n_words': 349}