<commit_before>(ns clj_hypergraph_db.model_parsing_functions
  (:use [clojure.tools.logging :only (info)]))


(defn def-attribute
  ""
  [name]
 (do
   (info "resolving attributtes")
   {:type :attribute :name name}))


(defn merge-attributes
  ""
  [& attributes]
  (reduce
    (fn
      [attribute-one attribute-two]
      (reduce (fn
                [previous-map key-to-merge-in]
                (assoc previous-map key-to-merge-in (merge (previous-map key-to-merge-in) (attribute-one key-to-merge-in))))
              (merge attribute-one attribute-two)
              (filter #(contains? (keys attribute-one) %) (keys attribute-two))))
    {} attributes))


(defn def-type
  ""
  [name & attributes]
  (apply merge (cons {:type :type :name name} (merge-attributes attributes))))


(defn def-link
  ""
  [name & attributes]
  (apply merge (cons {:type :link :name name} (merge-attributes attributes))))


(defn with-attribute
  ""
  [attribute]
  {:attribute (list attribute)})


(defn from
  ""
  [attribute]
  {:from (list attribute)})


(defn to
  ""
  [attribute]
  {:to (list attribute)})


(defn from-to
  ""
  [attribute]
  {:from (list attribute) :to (list attribute)})


(defn parse
  ""
  [file]
  (do
    (info "starts parsing")
    (info (fn? def-attribute))
    (info (fn? def-type))
    ;(info (def-attribute :Abstract))
    ;(map #(println (str %)) (read-string
    (info
          (map
            (fn
              [token]
              (clojure.string/replace text (str token) (str "clj_hypergraph_db.model_parsing_functions/" token)))
            file))
           ; (remove #{'parse} (keys (ns-publics 'clj_hypergraph_db.model_parsing_functions)))))





    (comment map (fn [arg] (eval (list 'do '(use 'clj_hypergraph_db.model_parsing_functions) arg))) file)

    ;(map #(binding [*ns* clj_hypergraph_db.model_parsing_functions] (eval %)) file)
    ;(eval (second file))
    ;(map #(eval %) file)
    ;(apply merge (map #(eval %) file))
  ))<commit_msg>reverted to previous reduce attempt<commit_after>(ns clj_hypergraph_db.model_parsing_functions
  (:use [clojure.tools.logging :only (info)]))


(defn def-attribute
  ""
  [name]
 (do
   (info "resolving attributtes")
   {:type :attribute :name name}))


(defn merge-attributes
  ""
  [& attributes]
  (reduce
    (fn
      [attribute-one attribute-two]
      (reduce (fn
                [previous-map key-to-merge-in]
                (assoc previous-map key-to-merge-in (merge (previous-map key-to-merge-in) (attribute-one key-to-merge-in))))
              (merge attribute-one attribute-two)
              (filter #(contains? (keys attribute-one) %) (keys attribute-two))))
    {} attributes))


(defn def-type
  ""
  [name & attributes]
  (apply merge (cons {:type :type :name name} (merge-attributes attributes))))


(defn def-link
  ""
  [name & attributes]
  (apply merge (cons {:type :link :name name} (merge-attributes attributes))))


(defn with-attribute
  ""
  [attribute]
  {:attribute (list attribute)})


(defn from
  ""
  [attribute]
  {:from (list attribute)})


(defn to
  ""
  [attribute]
  {:to (list attribute)})


(defn from-to
  ""
  [attribute]
  {:from (list attribute) :to (list attribute)})


(defn parse
  ""
  [file]
  (do
    (info "starts parsing")
    (info (fn? def-attribute))
    (info (fn? def-type))
    ;(info (def-attribute :Abstract))
    ;(map #(println (str %)) (read-string
    (info
          (reduce
            (fn
              [token]
              (clojure.string/replace text (str token) (str "clj_hypergraph_db.model_parsing_functions/" token)))
            file
           (remove #{'parse} (keys (ns-publics 'clj_hypergraph_db.model_parsing_functions)))))





    (comment map (fn [arg] (eval (list 'do '(use 'clj_hypergraph_db.model_parsing_functions) arg))) file)

    ;(map #(binding [*ns* clj_hypergraph_db.model_parsing_functions] (eval %)) file)
    ;(eval (second file))
    ;(map #(eval %) file)
    ;(apply merge (map #(eval %) file))
  ))
{'fraction_non_alphanumeric': 0.15807991957778336, 'fraction_numerical': 0.0, 'mean_word_length': 2.8715953307392996, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7653309', 'n_tokens_mistral': 1398, 'n_tokens_neox': 1341, 'n_words': 332}