<commit_before>(ns omhome.transformations
  (:require [clojure.java.io :as io]
            [clojure.string :as s]
            [clygments.core :as pygments]
            [net.cgrand.enlive-html :as enlive])
  (:import (java.io StringReader)))

(defn- unwrap-pygments-code [highlighted]
  "Fix Pygments' over-zealous wrapping of code in <div> and <pre> tags, already present in our content"
  (-> highlighted
      StringReader.
      enlive/html-resource
      (enlive/select [:pre])
      first
      :content))

(defn- highlight [enlive-node]
  "Apply Pygments code highlighting to the supplied enlive node, picking up the desired language from applied CSS"
  (let [code (->> enlive-node :content (apply str))
        lang (->> enlive-node :attrs :class keyword)]
    (assoc enlive-node :content (-> code
                                    (pygments/highlight lang :html)
                                    unwrap-pygments-code))))

(defn highlight-code-blocks [html]
  "Extract code blocks and apply code-highlighting to them."
  (enlive/sniptest html
                   [:pre :code] highlight
                   [:pre :code] #(assoc-in % [:attrs :class] "codehilite")))

(defn- includes? [s] (enlive/text-pred #(s/includes? % s)))
(defn- re= [s] (enlive/text-pred #(boolean (re-find s %))))

(def ellipsis "...")
(def triple-prime "'''")
(def double-prime "''")
(def opening-double-quote #"(\W|^)\"(\S)")
(def closing-double-quote #"(\u201c[^\"]*)\"([^\"]*$|[^\u201c\"]*\u201c)")
(def closing-double-quote-supp #"([^0-9])\"")
(def opening-apostrophe #"(\W|^)'(\S)")
(def possessive-apostrophe #"i?([a-z])'([a-z])")
(def closing-apostrophe #"i?((\u2018[^']*)|[a-z])'([^0-9]|$)")
(def backwards-apostrophe #"i?(\B|^)\u2018(?=([^\u2019]*\u2019\b)*([^\u2019\u2018]*\W[\u2019\u2018]\b|[^\u2019\u2018]*$))")
(def shortened-years-apostrophe #"i?(\u2018)([0-9]{2}[^\u2019]*)(\u2018([^0-9]|$)|$|\u2019[a-z])")
(def standalone-apostrophe "'")

(defn smartypants [html]
  "Convert straight quotes to curly, whether single or double, and your sets of three periods to ellipses, and, verily, your multiple dashes to their rightful selves."
  (enlive/sniptest html
                   [(includes? ellipsis)] #(s/replace % ellipsis "\u2026")
                   [(includes? triple-prime)] #(s/replace % triple-prime "\u2034")
                   [(includes? double-prime)] #(s/replace % double-prime "\u2033")
                   [(re= opening-double-quote)] #(s/replace % opening-double-quote "$1\u201c$2")
                   [(re= closing-double-quote)] #(s/replace % closing-double-quote "$1\u201d$2")
                   [(re= closing-double-quote-supp)] #(s/replace % closing-double-quote-supp "$1\u201d")
                   [(re= opening-apostrophe)] #(s/replace % opening-apostrophe "$1\u2018$2")
                   [(re= possessive-apostrophe)] #(s/replace % possessive-apostrophe "$1\u2019$2")
                   [(re= closing-apostrophe)] #(s/replace % closing-apostrophe "$1\u2019$3")
                   [(re= shortened-years-apostrophe)] #(s/replace % shortened-years-apostrophe "\u2019$2$3")
                   [(re= backwards-apostrophe)] #(s/replace % backwards-apostrophe "$1\u2019")
                   [(includes? standalone-apostrophe)] #(s/replace % standalone-apostrophe "\u2032")
                   ))
<commit_msg>Em & en dashes now converted<commit_after>(ns omhome.transformations
  (:require [clojure.java.io :as io]
            [clojure.string :as s]
            [clygments.core :as pygments]
            [net.cgrand.enlive-html :as enlive])
  (:import (java.io StringReader)))

(defn- unwrap-pygments-code [highlighted]
  "Fix Pygments' over-zealous wrapping of code in <div> and <pre> tags, already present in our content"
  (-> highlighted
      StringReader.
      enlive/html-resource
      (enlive/select [:pre])
      first
      :content))

(defn- highlight [enlive-node]
  "Apply Pygments code highlighting to the supplied enlive node, picking up the desired language from applied CSS"
  (let [code (->> enlive-node :content (apply str))
        lang (->> enlive-node :attrs :class keyword)]
    (assoc enlive-node :content (-> code
                                    (pygments/highlight lang :html)
                                    unwrap-pygments-code))))

(defn highlight-code-blocks [html]
  "Extract code blocks and apply code-highlighting to them."
  (enlive/sniptest html
                   [:pre :code] highlight
                   [:pre :code] #(assoc-in % [:attrs :class] "codehilite")))

(defn- includes? [s] (enlive/text-pred #(s/includes? % s)))
(defn- re= [s] (enlive/text-pred #(boolean (re-find s %))))

(def ellipsis "...")
(def triple-prime "'''")
(def double-prime "''")
(def em-dash "---")
(def en-dash "--")
(def opening-double-quote #"(\W|^)\"(\S)")
(def closing-double-quote #"(\u201c[^\"]*)\"([^\"]*$|[^\u201c\"]*\u201c)")
(def closing-double-quote-supp #"([^0-9])\"")
(def opening-apostrophe #"(\W|^)'(\S)")
(def possessive-apostrophe #"i?([a-z])'([a-z])")
(def closing-apostrophe #"i?((\u2018[^']*)|[a-z])'([^0-9]|$)")
(def backwards-apostrophe #"i?(\B|^)\u2018(?=([^\u2019]*\u2019\b)*([^\u2019\u2018]*\W[\u2019\u2018]\b|[^\u2019\u2018]*$))")
(def shortened-years-apostrophe #"i?(\u2018)([0-9]{2}[^\u2019]*)(\u2018([^0-9]|$)|$|\u2019[a-z])")
(def standalone-apostrophe "'")

(defn smartypants [html]
  "Convert straight quotes to curly, whether single or double, and your sets of three periods to ellipses, and, verily, your multiple dashes to their rightful selves."
  (enlive/sniptest html
                   [(includes? ellipsis)] #(s/replace % ellipsis "\u2026")
                   [(includes? triple-prime)] #(s/replace % triple-prime "\u2034")
                   [(includes? double-prime)] #(s/replace % double-prime "\u2033")
                   [(includes? em-dash)] #(s/replace % em-dash "\u2014")
                   [(includes? en-dash)] #(s/replace % en-dash "\u2013")
                   [(re= opening-double-quote)] #(s/replace % opening-double-quote "$1\u201c$2")
                   [(re= closing-double-quote)] #(s/replace % closing-double-quote "$1\u201d$2")
                   [(re= closing-double-quote-supp)] #(s/replace % closing-double-quote-supp "$1\u201d")
                   [(re= opening-apostrophe)] #(s/replace % opening-apostrophe "$1\u2018$2")
                   [(re= possessive-apostrophe)] #(s/replace % possessive-apostrophe "$1\u2019$2")
                   [(re= closing-apostrophe)] #(s/replace % closing-apostrophe "$1\u2019$3")
                   [(re= shortened-years-apostrophe)] #(s/replace % shortened-years-apostrophe "\u2019$2$3")
                   [(re= backwards-apostrophe)] #(s/replace % backwards-apostrophe "$1\u2019")
                   [(includes? standalone-apostrophe)] #(s/replace % standalone-apostrophe "\u2032")
                   ))

{'fraction_non_alphanumeric': 0.19391634980988592, 'fraction_numerical': 0.04007019596373208, 'mean_word_length': 2.7027612344342176, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4914497', 'n_tokens_mistral': 2626, 'n_tokens_neox': 2297, 'n_words': 500}