<commit_before>(ns hydrox.meta.util
  (:require [clojure.zip :as zip]
            [clojure.string :as string]
            [rewrite-clj.zip :as source]
            [rewrite-clj.node :as node]
            [clojure.java.io :as io]))

(defn append-node
  "Adds node as well as whitespace and newline on right
 
   (-> (z/of-string \"(+)\")
       (z/down)
       (append-node 2)
       (append-node 1)
       (z/->root-string))
   => (+
  1
   2)"
  {:added "0.1"}
  [zloc node]
  (if node
    (-> zloc
        (zip/insert-right node)
        (zip/insert-right (node/whitespace-node "  "))
        (zip/insert-right (node/newline-node "\n")))
    zloc))

(defn strip-quotes
  "gets rid of quotes in a string
 
   (strip-quotes \"\\\"hello\\\"\")
   => hello"
  {:added "0.1"}
  [s]
  (if (and (.startsWith s "\"")
           (.endsWith s "\""))
    (subs s 1 (dec (count s)))
    s))

(defn escape-escapes
  [s]
  (-> s
      (.replaceAll "(\\\\)([A-Za-z])" "$1$1$2")))

(defn escape-quotes
  "makes sure that quotes are printable in string form
 
   (escape-quotes \"\\\"hello\\\"\")
   => \\\"hello\\\""
  {:added "0.1"}
  [s]
  (-> s
      (.replaceAll "(\\\\)?\"" "$1$1\\\\\\\"")))

(defn nodes->docstring
  "converts nodes to a docstring compatible
   (->> (z/of-string \"\\\"hello\\\"\\n  (+ 1 2)\\n => 3 \")
        (iterate z/right*)
        (take-while identity)
        (map z/node)
        (nodes->docstring)
        (node/string))
   => \"hello
   (+ 1 2)
   => 3 \"
 
   (->> (z/of-string \"[\\e \\d]\")
        (iterate z/right*)
        (take-while identity)
        (map z/node)
        (nodes->docstring)
       (node/string))
   => \"[\\e \\d]\""
  {:added "0.1"}
  [nodes]
  (->> nodes
       (map node/string)
       (map strip-quotes)
       (string/join)
       (escape-escapes)
       (escape-quotes)
       (string/split-lines)
       (map-indexed (fn [i s]
                      (str (if-not (or (zero? i)
                                       (= i (dec (count nodes))))
                             " ")
                           s)))
       (node/string-node)))

(defn import-location
  "imports the meta information and docstring"
  {:added "0.1"}
  [zloc nsp gathered]
  (let [sym   (source/sexpr zloc)
        nodes (get-in gathered [nsp sym :docs])
        meta  (get-in gathered [nsp sym :meta])]
    (-> zloc
        (append-node meta)
        (append-node (if nodes (nodes->docstring nodes))))))

(defn write-to-file
  "exports the zipper contents to file"
  {:added "0.1"}
  [zloc file]
  (->> (iterate source/right* zloc)
       (take-while identity)
       (map source/node)
       (map node/string)
       (string/join)
       (spit file)))

(defn all-files
  "finds all files in the project given a context
 
   (->> (all-files {:root (.getCanonicalPath (io/file \"example\"))} :root \".md\")
        (map #(.getName %)))
   => [\"README.md\"]"
  {:added "0.1"}
  [project path-type extension]
  (->> project
       path-type
       (#(if (sequential? %) % [%]))
       (map (fn [dir] (file-seq (io/file (if (.startsWith dir "/")
                                              dir
                                              (str (:root project) "/" dir))))))
       (filter (fn [file] (and (.isFile file)
                               (.endsWith (str file) extension))))
       (map #(.getCanonicalFile %))))<commit_msg>fixing test<commit_after>(ns hydrox.meta.util
  (:require [clojure.zip :as zip]
            [clojure.string :as string]
            [rewrite-clj.zip :as source]
            [rewrite-clj.node :as node]
            [clojure.java.io :as io]))

(defn append-node
  "Adds node as well as whitespace and newline on right
 
   (-> (z/of-string \"(+)\")
       (z/down)
       (append-node 2)
       (append-node 1)
       (z/->root-string))
   => (+
  1
   2)"
  {:added "0.1"}
  [zloc node]
  (if node
    (-> zloc
        (zip/insert-right node)
        (zip/insert-right (node/whitespace-node "  "))
        (zip/insert-right (node/newline-node "\n")))
    zloc))

(defn strip-quotes
  "gets rid of quotes in a string
 
   (strip-quotes \"\\\"hello\\\"\")
   => hello"
  {:added "0.1"}
  [s]
  (if (and (.startsWith s "\"")
           (.endsWith s "\""))
    (subs s 1 (dec (count s)))
    s))

(defn escape-escapes
  [s]
  (-> s
      (.replaceAll "(\\\\)([A-Za-z])" "$1$1$2")))

(defn escape-quotes
  "makes sure that quotes are printable in string form
 
   (escape-quotes \"\\\"hello\\\"\")
   => \\\"hello\\\""
  {:added "0.1"}
  [s]
  (-> s
      (.replaceAll "(\\\\)?\"" "$1$1\\\\\\\"")))

(defn nodes->docstring
  "converts nodes to a docstring compatible
   (->> (z/of-string \"\\\"hello\\\"\\n  (+ 1 2)\\n => 3 \")
        (iterate z/right*)
        (take-while identity)
        (map z/node)
        (nodes->docstring)
        (node/string))
   => \"hello
   (+ 1 2)
   => 3 \"
 
   (->> (z/of-string \"[\\e \\d]\")
        (iterate z/right*)
        (take-while identity)
        (map z/node)
        (nodes->docstring)
       (node/string))
   => \"[\\e \\d]\""
  {:added "0.1"}
  [nodes]
  (->> nodes
       (map node/string)
       (map strip-quotes)
       (string/join)
       (escape-escapes)
       (escape-quotes)
       (string/split-lines)
       (map-indexed (fn [i s]
                      (str (if-not (or (zero? i)
                                       (= i (dec (count nodes))))
                             " ")
                           s)))
       (node/string-node)))

(defn import-location
  "imports the meta information and docstring"
  {:added "0.1"}
  [zloc nsp gathered]
  (let [sym   (source/sexpr zloc)
        nodes (get-in gathered [nsp sym :docs])
        meta  (get-in gathered [nsp sym :meta])]
    (-> zloc
        (append-node meta)
        (append-node (if nodes (nodes->docstring nodes))))))

(defn write-to-file
  "exports the zipper contents to file"
  {:added "0.1"}
  [zloc file]
  (->> (iterate source/right* zloc)
       (take-while identity)
       (map source/node)
       (map node/string)
       (string/join)
       (spit file)))

(defn all-files
  "finds all files in the project given a context
 
   (->> (all-files {:root (.getCanonicalPath (io/file \"example\"))} :root \".md\")
        (map #(.getName %)))
   => [\"README.md\"]"
  {:added "0.1"}
  [project path-type extension]
  (->> project
       path-type
       (#(if (sequential? %) % [%]))
       (mapcat (fn [dir] (file-seq (io/file (if (.startsWith dir "/")
                                              dir
                                              (str (:root project) "/" dir))))))
       (filter (fn [file] (and (.isFile file)
                               (.endsWith (str file) extension))))
       (map #(.getCanonicalFile %))))
{'fraction_non_alphanumeric': 0.1937211724445767, 'fraction_numerical': 0.008927242969796162, 'mean_word_length': 1.62578125, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7286692', 'n_tokens_mistral': 2454, 'n_tokens_neox': 2349, 'n_words': 581}