<commit_before>(ns hyjinks.core
  (:use [clojure.string :only (escape split join capitalize lower-case trim)]))

;; Forward definitions to resolve circular references

(declare tag?)
(declare extend-tag)

;; General helpers

(defn- prune-map [m]
  (let [rem? #(or (nil? %) (= "" %) (and (coll? %) (empty? %)))]
    (apply dissoc m (map first (filter (comp rem? second) m)))))

(defn- str-join [& items] (join "" (flatten items)))

(defn- interposep [sep pred coll]
  (cond
    (empty? coll) (empty coll)
    (= 1 (count coll)) coll
    :else (if (pred (first coll) (second coll))
      (concat [(first coll) sep] (interposep sep pred (rest coll)))
      (concat [(first coll)] (interposep sep pred (rest coll))))))

(defn- str-join-extra-spaces [& items]
  (->> items
    flatten
    (interposep " " #(not (or (tag? %1) (tag? %2))))
    (apply str)))

(def ^{:private true} escape-chars {
  \< "&lt;"
  \> "&gt;"
  \& "&amp;"
  \" "&quot;"
  \' "&#39;"})

(defn- html-escape [x]<commit_msg>Minor cleanup on (interposep)<commit_after>(ns hyjinks.core
  (:use [clojure.string :only (escape split join capitalize lower-case trim)]))

;; Forward definitions to resolve circular references

(declare tag?)
(declare extend-tag)

;; General helpers

(defn- prune-map [m]
  (let [rem? #(or (nil? %) (= "" %) (and (coll? %) (empty? %)))]
    (apply dissoc m (map first (filter (comp rem? second) m)))))

(defn- str-join [& items] (join "" (flatten items)))

(defn- interposep [sep pred coll]
  (cond
    (empty? coll)                     (empty coll)
    (= 1 (count coll))                coll
    (pred (first coll) (second coll)) (concat [(first coll) sep] (interposep sep pred (rest coll)))
    :else                             (concat [(first coll)] (interposep sep pred (rest coll)))))

(defn- str-join-extra-spaces [& items]
  (->> items
    flatten
    (interposep " " #(not (or (tag? %1) (tag? %2))))
    (apply str)))

(def ^{:private true} escape-chars {
  \< "&lt;"
  \> "&gt;"
  \& "&amp;"
  \" "&quot;"
  \' "&#39;"})

(defn- html-escape [x]
{'fraction_non_alphanumeric': 0.2013685239491691, 'fraction_numerical': 0.004887585532746823, 'mean_word_length': 3.0776892430278884, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '333996', 'n_tokens_mistral': 770, 'n_tokens_neox': 734, 'n_words': 227}