<commit_before>(ns selmer.filter-parser
  "Accessors are separated by dots like {{ foo.bar.0 }}
which gets translated into (get-in context-map [:foo :bar 0]). So you
can nest vectors and maps in your context-map.

Filters can be applied by separating then from the accessor
with pipes: {{ foo|lower|capitalize }}. They are applied one after
the other from left to right. Arguments can be passed to a filter
separated by colons: {{ foo|pluralize:y:ies }}. If an argument includes
spaces you can enclose it with doublequotes or colons: {{ foo|join:\", \" }}.

You can escape doublequotes inside doublequotes. And you can put colons
inside doublequotes which will be ignored for the purpose of separating
arguments."
  (:require [selmer.filters :refer [get-filter]]
            [selmer.util :refer [exception *escape-variables*]]
            [clojure.string :as s]))

;;; More Utils
(defn escape-html*
  [^String s]
  "HTML-escapes the given string. Escapes the same characters as django's escape."
  ;; This method is "Java in Clojure" for serious speedups.
  ;; Stolen from davidsantiago/quoin and modified.
  (if *escape-variables*
    (let [sb (StringBuilder.)
          slength (count s)]
      (loop [idx 0]
        (if (>= idx slength)
          (.toString sb)
          (let [c (char (.charAt s idx))]
            (case c
              \& (.append sb "&amp;")
              \< (.append sb "&lt;")
              \> (.append sb "&gt;")
              \" (.append sb "&quot;")
              \' (.append sb "&#39;")
              (.append sb c))
            (recur (inc idx))))))
    s))

(defn strip-doublequotes
  "Removes doublequotes from the start and end of a string if any."
  [^String s]
  (if (and (> (count s) 1)
           (= \" (first s) (.charAt s (dec (count s)))))
    (.substring s 1 (dec (count s)))
    s))

(defn escape-html
  "Must have the form [:safe safe-string] to prevent escaping. Alternatively,
  you can call selmer.util/turn-off-escaping! to turn it off completely."
  [x]
  (if (and (vector? x)
           (= :safe (first x)))
    (second x)
    (let [s (str x)]
      (escape-html* s))))

;;; Compile filters
(defn fix-accessor
  "Turns strings into keywords and strings like \"0\" into Longs
so it can access vectors as well as maps."
  [ks]
  (mapv (fn [^String s]
          (try (Long/valueOf s)
               (catch NumberFormatException _
                 (keyword s))))
        ks))

(defn split-filter-val
  "Split accessors like foo.bar.baz by the dot.
   But if there is a double dot '..' then it will leave it"
  [s]
  (let [ks (s/split s #"(?<!\.)\.(?!\.)")]
    (fix-accessor ks)))

(defn fix-filter-args
  "Map any sort of needed fixes to the arguments before passing them
to the filters. Only strips enclosing doublequotes for now."
  [args]
  ;; TODO - figure out what kind of extra args filters can take
  (map (fn [^String s]
         (strip-doublequotes s))
       args))

(defn lookup-args
  "Given a context map, return a function that accepts a filter
  argument and if it begins with @, return the value from the
  context map instead of treating it as a literal."
  [context-map]
  (fn [^String arg]
    (if (and (> (count arg) 1) (.startsWith arg "@"))
      (let [accessor (split-filter-val (subs arg 1))]
        (get-in context-map accessor arg))
      arg)))

(defn filter-str->fn<commit_msg>filter-parser split-filter-val now removes the double dot once the split has taken place<commit_after>(ns selmer.filter-parser
  "Accessors are separated by dots like {{ foo.bar.0 }}
which gets translated into (get-in context-map [:foo :bar 0]). So you
can nest vectors and maps in your context-map.

Filters can be applied by separating then from the accessor
with pipes: {{ foo|lower|capitalize }}. They are applied one after
the other from left to right. Arguments can be passed to a filter
separated by colons: {{ foo|pluralize:y:ies }}. If an argument includes
spaces you can enclose it with doublequotes or colons: {{ foo|join:\", \" }}.

You can escape doublequotes inside doublequotes. And you can put colons
inside doublequotes which will be ignored for the purpose of separating
arguments."
  (:require [selmer.filters :refer [get-filter]]
            [selmer.util :refer [exception *escape-variables*]]
            [clojure.string :as s]))

;;; More Utils
(defn escape-html*
  [^String s]
  "HTML-escapes the given string. Escapes the same characters as django's escape."
  ;; This method is "Java in Clojure" for serious speedups.
  ;; Stolen from davidsantiago/quoin and modified.
  (if *escape-variables*
    (let [sb (StringBuilder.)
          slength (count s)]
      (loop [idx 0]
        (if (>= idx slength)
          (.toString sb)
          (let [c (char (.charAt s idx))]
            (case c
              \& (.append sb "&amp;")
              \< (.append sb "&lt;")
              \> (.append sb "&gt;")
              \" (.append sb "&quot;")
              \' (.append sb "&#39;")
              (.append sb c))
            (recur (inc idx))))))
    s))

(defn strip-doublequotes
  "Removes doublequotes from the start and end of a string if any."
  [^String s]
  (if (and (> (count s) 1)
           (= \" (first s) (.charAt s (dec (count s)))))
    (.substring s 1 (dec (count s)))
    s))

(defn escape-html
  "Must have the form [:safe safe-string] to prevent escaping. Alternatively,
  you can call selmer.util/turn-off-escaping! to turn it off completely."
  [x]
  (if (and (vector? x)
           (= :safe (first x)))
    (second x)
    (let [s (str x)]
      (escape-html* s))))

;;; Compile filters
(defn fix-accessor
  "Turns strings into keywords and strings like \"0\" into Longs
so it can access vectors as well as maps."
  [ks]
  (mapv (fn [^String s]
          (try (Long/valueOf s)
               (catch NumberFormatException _
                 (keyword s))))
        ks))

(defn split-filter-val
  "Split accessors like foo.bar.baz by the dot.
   But if there is a double dot '..' then it will leave it"
  [s]
  (let [ks (clojure.string/split s #"(?<!\.)\.(?!\.)")
        kss (map
              (fn [s] (clojure.string/replace s ".." ".")) ks)] ;we remove the double dot here
    (fix-accessor kss)))

(defn fix-filter-args
  "Map any sort of needed fixes to the arguments before passing them
to the filters. Only strips enclosing doublequotes for now."
  [args]
  ;; TODO - figure out what kind of extra args filters can take
  (map (fn [^String s]
         (strip-doublequotes s))
       args))

(defn lookup-args
  "Given a context map, return a function that accepts a filter
  argument and if it begins with @, return the value from the
  context map instead of treating it as a literal."
  [context-map]
  (fn [^String arg]
    (if (and (> (count arg) 1) (.startsWith arg "@"))
      (let [accessor (split-filter-val (subs arg 1))]
        (get-in context-map accessor arg))
      arg)))

(defn filter-str->fn
{'fraction_non_alphanumeric': 0.12717265353418308, 'fraction_numerical': 0.0028968713789107765, 'mean_word_length': 2.561113976276431, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5498174', 'n_tokens_mistral': 2234, 'n_tokens_neox': 2128, 'n_words': 870}