<commit_before>(ns vip.data-processor.validation.xml
  (:require [clojure.data.xml :as xml]
            [clojure.java.io :as io]
            [vip.data-processor.db.sqlite :as sqlite]
            [vip.data-processor.validation.data-spec :as data-spec]))

(declare element->map)

(defn node->key-value [{:keys [tag content] :as node}]
  (let [tag (name tag)]
    (cond
      (empty? content)
      [tag nil]

      (every? (partial instance? clojure.data.xml.Element) content)
      [tag (element->map node)]

      :else
      [tag (first content)])))

(defn element->map [{:keys [tag attrs content]}]
  (let [id (:id attrs)]
    (into (if id {"id" id} {})
          (map node->key-value content))))

(defn validate-format-rules [ctx rows {:keys [tag-name columns]}]
  (let [format-rules (data-spec/create-format-rules tag-name columns)]
    (reduce (fn [ctx row]
              (data-spec/apply-format-rules format-rules ctx row (:id row)))
            ctx rows)))

(defn load-elements [ctx elements]
  (let [tag (:tag (first elements))]
    (if-let [data-spec (first (filter #(= tag (:tag-name %)) data-spec/data-specs))]
      (let [element-maps (map element->map elements)
            table-key (:table data-spec)
            sql-table (get-in ctx [:tables table-key])
            ctx (validate-format-rules ctx element-maps data-spec)
            columns (:columns data-spec)
            column-names (map :name columns)
            contents (map #(select-keys % column-names) element-maps)
            transforms (apply comp (data-spec/translation-fns columns))
            transformed-contents (map transforms contents)]
        (sqlite/bulk-import transformed-contents sql-table)
        ctx)
      (assoc-in ctx [:critical :xml-import tag :unknown] tag))))

(defn partition-by-n
  "Applies f to each value in coll, splitting it each time f returns a
  new value up to a maximum of n elements.  Returns a lazy seq of
  partitions."
  [f n coll]
  (lazy-seq
   (when-let [s (seq coll)]
     (let [fst (first s)
           fv (f fst)
           next-n (take (dec n) (next s))
           run (cons fst (take-while #(= fv (f %)) next-n))]
       (cons run (partition-by-n f n (seq (drop (count run) s))))))))

(defn load-xml [ctx]
  (let [xml-file (first (:input ctx))]
    (reduce load-elements ctx (partition-by-n :tag 100 (:content (xml/parse (io/reader xml-file)))))))
<commit_msg>[CS WR] Load data from XML attributes<commit_after>(ns vip.data-processor.validation.xml
  (:require [clojure.data.xml :as xml]
            [clojure.java.io :as io]
            [clojure.walk :refer [stringify-keys]]
            [vip.data-processor.db.sqlite :as sqlite]
            [vip.data-processor.validation.data-spec :as data-spec]))

(declare element->map)

(defn node->key-value [{:keys [tag content] :as node}]
  (let [tag (name tag)]
    (cond
      (empty? content)
      [tag nil]

      (every? (partial instance? clojure.data.xml.Element) content)
      [tag (element->map node)]

      :else
      [tag (first content)])))

(defn element->map [{:keys [attrs content]}]
  (into (stringify-keys attrs)
        (map node->key-value content)))

(defn validate-format-rules [ctx rows {:keys [tag-name columns]}]
  (let [format-rules (data-spec/create-format-rules tag-name columns)]
    (reduce (fn [ctx row]
              (data-spec/apply-format-rules format-rules ctx row (:id row)))
            ctx rows)))

(defn load-elements [ctx elements]
  (let [tag (:tag (first elements))]
    (if-let [data-spec (first (filter #(= tag (:tag-name %)) data-spec/data-specs))]
      (let [element-maps (map element->map elements)
            table-key (:table data-spec)
            sql-table (get-in ctx [:tables table-key])
            ctx (validate-format-rules ctx element-maps data-spec)
            columns (:columns data-spec)
            column-names (map :name columns)
            contents (map #(select-keys % column-names) element-maps)
            transforms (apply comp (data-spec/translation-fns columns))
            transformed-contents (map transforms contents)]
        (sqlite/bulk-import transformed-contents sql-table)
        ctx)
      (assoc-in ctx [:critical :xml-import tag :unknown] tag))))

(defn partition-by-n
  "Applies f to each value in coll, splitting it each time f returns a
  new value up to a maximum of n elements.  Returns a lazy seq of
  partitions."
  [f n coll]
  (lazy-seq
   (when-let [s (seq coll)]
     (let [fst (first s)
           fv (f fst)
           next-n (take (dec n) (next s))
           run (cons fst (take-while #(= fv (f %)) next-n))]
       (cons run (partition-by-n f n (seq (drop (count run) s))))))))

(defn load-xml [ctx]
  (let [xml-file (first (:input ctx))]
    (reduce load-elements ctx (partition-by-n :tag 100 (:content (xml/parse (io/reader xml-file)))))))

{'fraction_non_alphanumeric': 0.14923204649232047, 'fraction_numerical': 0.0012453300124533001, 'mean_word_length': 2.5564575645756458, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3301135', 'n_tokens_mistral': 1585, 'n_tokens_neox': 1549, 'n_words': 482}