<commit_before>(ns kixi.hecuba.api.parse-test
  (:use clojure.test)
  (:require [clojure.test.check.generators :as gen]
            [clojure.test.check :as tc]
            [schema.core :as s]
            [schema_gen.core :as sg]
            [clojure.test.check.properties :as prop]))

(defn non-empty? [n] (if (coll? n) (seq n) n))

;; This is a function copied from new_profile.cljs. Test shoudl be moved to cljs tests once they are set up.
(defn parse
  "Remove all empty elements from the nested data structure and flatten :_value elements."
  [cursor]
  (clojure.walk/postwalk (fn [m]
                          (cond
                           (:_value m)
                           (:_value m)

                           (map? m)
                           (reduce-kv (fn [agg k v] (if (non-empty? v) (assoc agg k v) agg)) {} m)

                           (and (coll? m) (not (keyword? (first m))))
                           (into (empty m) (filter non-empty? m))

                           :else
                           m))<commit_msg>Fix typo.<commit_after>(ns kixi.hecuba.api.parse-test
  (:use clojure.test)
  (:require [clojure.test.check.generators :as gen]
            [clojure.test.check :as tc]
            [schema.core :as s]
            [schema_gen.core :as sg]
            [clojure.test.check.properties :as prop]))

(defn non-empty? [n] (if (coll? n) (seq n) n))

;; This is a function copied from new_profile.cljs. Test should be moved to cljs tests once they are set up.
(defn parse
  "Remove all empty elements from the nested data structure and flatten :_value elements."
  [cursor]
  (clojure.walk/postwalk (fn [m]
                          (cond
                           (:_value m)
                           (:_value m)

                           (map? m)
                           (reduce-kv (fn [agg k v] (if (non-empty? v) (assoc agg k v) agg)) {} m)

                           (and (coll? m) (not (keyword? (first m))))
                           (into (empty m) (filter non-empty? m))

                           :else
                           m))
{'fraction_non_alphanumeric': 0.11711281070745698, 'fraction_numerical': 0.0, 'mean_word_length': 1.4029850746268657, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '498918', 'n_tokens_mistral': 649, 'n_tokens_neox': 634, 'n_words': 206}