<commit_before>(ns domshot.core
  (:require [goog.dom]))

(enable-console-print!)


;; ---- Node validators ---- ;;

(def node-invalidators
  "An atom of a list of predicates. Each takes the current node as its only arg.
  Predicates should return `true` if you want `snapshot` to ignore the node."
  (atom ()))

(defn include-node? [node]
  (not-any? false? (map #(% node) @node-invalidators)))

(def attr-invalidators
  "An atom of a list of predicates. Each takes an attr name and val as its
  args. Predicates should return `true` if you want `snapshot` to ignore the
  attr name-val pair."
  (atom ()))

(defn include-attr? [name val]
  (not-any? false? (map #(% name val) @attr-invalidators)))

(defmulti add-invalidator (fn [_ invalidator-type] invalidator-type))

(defmethod add-invalidator :attribute [fn]
  (swap! attr-invalidators conj fn))

(defmethod add-invalidator :default [fn]
  (swap! attr-invalidators conj fn))


;; ---- Serializing the dom ---- ;;

(defn get-attrs [node]
  (if (.-attributes node)
    (into {}
          (let [attrs (.-attributes node)]
            (for [i (range (.-length attrs))
                  :let [attr  (aget attrs i)
                        name  (.-name attr)
                        value (.-value attr)]
                  :when (include-attr? name value)]
              [name value])))
    {}))

(defn handle-special-nodes
  "Fills in important node-specific key-val pairs"
  [node-map native-node]
  (let [node-types goog.dom/NodeType]
    (condp = (.-nodeType native-node)
      (.-TEXT node-types) (assoc node-map :text (.-nodeValue native-node))
      node-map)))

(defn gen-basic-node-map [node]
  {:node-name  (.-nodeName node)
   :attributes (get-attrs node)})

(defn snapshot<commit_msg>change invalidator to qualifier to simplify logic<commit_after>(ns domshot.core
  (:require [goog.dom]))

(enable-console-print!)


;; ---- Qualifiers ---- ;;

(def node-qualifiers
  "An atom of a list of predicates. Each takes the current node as its only arg.
  All predicates should return `true` if you want `snapshot` to include the node."
  (atom ()))

(defn include-node? [node]
  (not-any? false? (map #(% node) @node-qualifiers)))

(def attr-qualifiers
  "An atom of a list of predicates. Each takes an attr name and val as its
  args. All predicates should return `true` if you want `snapshot` to include the
  attr name-val pair."
  (atom ()))

(defn include-attr? [name val]
  (not-any? false? (map #(% name val) @attr-qualifiers)))

(defmulti add-qualifier (fn [_ qualifier-type] qualifier-type))

(defmethod add-qualifier :attribute [fn]
  (swap! attr-qualifiers conj fn))

(defmethod add-qualifier :default [fn]
  (swap! attr-qualifiers conj fn))


;; ---- Serializing the dom ---- ;;

(defn get-attrs [node]
  (if (.-attributes node)
    (into {}
          (let [attrs (.-attributes node)]
            (for [i (range (.-length attrs))
                  :let [attr  (aget attrs i)
                        name  (.-name attr)
                        value (.-value attr)]
                  :when (include-attr? name value)]
              [name value])))
    {}))

(defn handle-special-nodes
  "Fills in important node-specific key-val pairs"
  [node-map native-node]
  (let [node-types goog.dom/NodeType]
    (condp = (.-nodeType native-node)
      (.-TEXT node-types) (assoc node-map :text (.-nodeValue native-node))
      node-map)))

(defn gen-basic-node-map [node]
  {:node-name  (.-nodeName node)
   :attributes (get-attrs node)})

(defn snapshot
{'fraction_non_alphanumeric': 0.146966676160638, 'fraction_numerical': 0.0, 'mean_word_length': 2.6431535269709543, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '772382', 'n_tokens_mistral': 1149, 'n_tokens_neox': 1123, 'n_words': 370}