<commit_before>(ns dalap.test.cljsrc-walk
  (:use [clojure.test :only [deftest is]])
  (:use [clojure.java.io :only [reader make-parents]])
  (:require [clojure.string :as string])
  (:import [clojure.lang LineNumberingPushbackReader])
  (:import [java.io File])

  (:use dalap.html.selector)
  (:use [dalap.walk :only [walk]]))

(defn visit-form [form w]
  ;; a modified version of clojure.walk with the ability to drop forms
  (letfn [(filter-map [f form] (remove #(= % ::drop) (map f form)))]
    (cond
      (list? form) (apply list (filter-map w form))
      (instance? clojure.lang.IMapEntry form) (vec (filter-map w form))
      (seq? form) (doall (filter-map w form))
      (coll? form) (into (empty form) (filter-map w form))
      :else form)))

(def clj-forms-to-drop #{'defmacro 'comment})
(def cljs-core-transform-rules
  [(fn [form w] (contains? (meta form) :cljs))
   (comp :cljs meta)
   ;;
   (fn [form w] (contains? (meta form) :clj))
   (constantly ::drop)
   ;;
   (fn [form w] (and (list? form) (clj-forms-to-drop (first form))))
   (constantly ::drop)])

(def cljs-type-transform-rules
  ['clojure.lang.Atom 'cljs.core.Atom
   'clojure.lang.PersistentVector 'cljs.core.PersistentVector
   'Object 'default

   'String 'string
   'java.lang.String 'string

   'Number 'number
   'java.lang.Number 'number

   'Error 'js/Error
   'Error. 'js/Error.])

(def cljs-default-transform-rules
  (concat cljs-core-transform-rules
          cljs-type-transform-rules))

(defn cljs-transform
  ([forms]
     (cljs-transform forms cljs-default-transform-rules))
  ([forms rules]
     (walk forms ((gen-decorator rules) visit-form))))

(defn read-file
  ;; see https://github.com/jonase/kibit/blob/master/src/kibit/check.clj
  "Gen a lazy sequence of top level forms from a LineNumberingPushbackReader"
  [^LineNumberingPushbackReader r]
  (lazy-seq
    (let [form (try
                 (read r false ::eof)
                 (catch Exception e
                   (throw (Exception.
                           (str "Dalap's reader crashed"
                                (.getMessage e)) e))))]
      (when-not (= form ::eof)
        (cons form (read-file r))))))

(defn read-clj?s [file-or-path]
  (read-file
   (LineNumberingPushbackReader.
    (java.io.StringReader.
     (string/replace (slurp file-or-path) #"\#_\(\s*cljs\b" "(do ")))))

(defn warning-str [orig-path]
  (str ";;This file autogenerated from \n;;\n;; " orig-path
       " @ " (java.util.Date.)
       "\n;;\n"))

(defn transform-file
  ([file-or-path rules-vector-fn-or-map]
     (let [f (File. file-or-path)
           path (.getPath f)
           rules (if (or (fn? rules-vector-fn-or-map)
                         (map? rules-vector-fn-or-map))
                   (rules-vector-fn-or-map path)
                   rules-vector-fn-or-map)
           munged-forms (cljs-transform (read-clj?s file-or-path) rules)]
       (str (warning-str path)
            (string/join "\n" munged-forms)))))

;;; ;Taken from clojure.tools.namespace
(defn clj-source-file?
  "Returns true if file is a normal file with a .clj extension."
  [^File file]
  (and (.isFile file)
       (.endsWith (.getName file) ".clj")))

(defn find-clj-sources-in-dir
  "Searches recursively under dir for CLJ files.
Returns a sequence of File objects, in breadth-first sort order."
  [^File dir]
  ;; Use sort by absolute path to get breadth-first search.
  (sort-by #(.getAbsolutePath ^File %)
           (filter clj-source-file? (file-seq dir))))

(defn transform-directory
  ([clj-path output-path]
     (transform-directory
      clj-path output-path "cljs" cljs-default-transform-rules
      (constantly true)))

  ([clj-path output-path extension rules file-predicate]
     (doseq [f (find-clj-sources-in-dir (File. clj-path))]
       (when (file-predicate f)
         (let [generated-f (File. (-> (.getPath f)
                                      (string/replace clj-path output-path)
                                      (string/replace #"clj$" extension)))]
           (make-parents generated-f)
           (spit generated-f (transform-file f rules)))))))

(comment
  (transform-directory "src/dalap" ".generated")
  (print (string/join
          "\n"
          (cljs-transform (read-clj?s
                           "test/dalap/test/cljsrc_walk_sample.clj")))))

;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(deftest test-source-transformation
  (let [rules (concat
               ['something 'something-else
                'dalap.html.selector 'dalap.cljs.selector
                ] cljs-core-transform-rules)
        transform #(cljs-transform % rules)
        ]
    (is (= (transform '(something ^{:cljs something-else-cljs}
                                  foobar))
           '(something-else something-else-cljs)))

    (is (= (transform '(abcd ^:clj foobar))
           '(abcd)))

    (is (= (transform '(ns dalap.test.html.selector
                         (:use clojure.test)
                         (:use [clojure.pprint :only (pprint)])

                         (:require [dalap.html :as html])

                         ^{:cljs (:require [dalap.cljs.defaults :as defaults])}
                         (:require [dalap.defaults :as defaults])

                         (:use dalap.html.selector)
                         (:use [dalap.walk :only [walk]])
                         (:use [dalap.html :only [add-class]])))
           '(ns dalap.test.html.selector
              (:use clojure.test)
              (:use [clojure.pprint :only (pprint)])

              (:require [dalap.html :as html])
              (:require [dalap.cljs.defaults :as defaults])
              (:use dalap.cljs.selector)
              (:use [dalap.walk :only [walk]])
              (:use [dalap.html :only [add-class]]))))))
<commit_msg>fix test for 32bit java (bad arity finding)<commit_after>(ns dalap.test.cljsrc-walk
  (:use [clojure.test :only [deftest is]])
  (:use [clojure.java.io :only [reader make-parents]])
  (:require [clojure.string :as string])
  (:import [clojure.lang LineNumberingPushbackReader])
  (:import [java.io File])

  (:use dalap.html.selector)
  (:use [dalap.walk :only [walk]]))

(defn visit-form [form w]
  ;; a modified version of clojure.walk with the ability to drop forms
  (letfn [(filter-map [f form] (remove #(= % ::drop) (map f form)))]
    (cond
      (list? form) (apply list (filter-map w form))
      (instance? clojure.lang.IMapEntry form) (vec (filter-map w form))
      (seq? form) (doall (filter-map w form))
      (coll? form) (into (empty form) (filter-map w form))
      :else form)))

(def clj-forms-to-drop #{'defmacro 'comment})
(def cljs-core-transform-rules
  [(fn [form w] (contains? (meta form) :cljs))
   (fn [form w_] ((comp :cljs meta) form))
   ;;
   (fn [form w] (contains? (meta form) :clj))
   (constantly ::drop)
   ;;
   (fn [form w] (and (list? form) (clj-forms-to-drop (first form))))
   (constantly ::drop)])

(def cljs-type-transform-rules
  ['clojure.lang.Atom 'cljs.core.Atom
   'clojure.lang.PersistentVector 'cljs.core.PersistentVector
   'Object 'default

   'String 'string
   'java.lang.String 'string

   'Number 'number
   'java.lang.Number 'number

   'Error 'js/Error
   'Error. 'js/Error.])

(def cljs-default-transform-rules
  (concat cljs-core-transform-rules
          cljs-type-transform-rules))

(defn cljs-transform
  ([forms]
     (cljs-transform forms cljs-default-transform-rules))
  ([forms rules]
     (walk forms ((gen-decorator rules) visit-form))))

(defn read-file
  ;; see https://github.com/jonase/kibit/blob/master/src/kibit/check.clj
  "Gen a lazy sequence of top level forms from a LineNumberingPushbackReader"
  [^LineNumberingPushbackReader r]
  (lazy-seq
    (let [form (try
                 (read r false ::eof)
                 (catch Exception e
                   (throw (Exception.
                           (str "Dalap's reader crashed"
                                (.getMessage e)) e))))]
      (when-not (= form ::eof)
        (cons form (read-file r))))))

(defn read-clj?s [file-or-path]
  (read-file
   (LineNumberingPushbackReader.
    (java.io.StringReader.
     (string/replace (slurp file-or-path) #"\#_\(\s*cljs\b" "(do ")))))

(defn warning-str [orig-path]
  (str ";;This file autogenerated from \n;;\n;; " orig-path
       " @ " (java.util.Date.)
       "\n;;\n"))

(defn transform-file
  ([file-or-path rules-vector-fn-or-map]
     (let [f (File. file-or-path)
           path (.getPath f)
           rules (if (or (fn? rules-vector-fn-or-map)
                         (map? rules-vector-fn-or-map))
                   (rules-vector-fn-or-map path)
                   rules-vector-fn-or-map)
           munged-forms (cljs-transform (read-clj?s file-or-path) rules)]
       (str (warning-str path)
            (string/join "\n" munged-forms)))))

;;; ;Taken from clojure.tools.namespace
(defn clj-source-file?
  "Returns true if file is a normal file with a .clj extension."
  [^File file]
  (and (.isFile file)
       (.endsWith (.getName file) ".clj")))

(defn find-clj-sources-in-dir
  "Searches recursively under dir for CLJ files.
Returns a sequence of File objects, in breadth-first sort order."
  [^File dir]
  ;; Use sort by absolute path to get breadth-first search.
  (sort-by #(.getAbsolutePath ^File %)
           (filter clj-source-file? (file-seq dir))))

(defn transform-directory
  ([clj-path output-path]
     (transform-directory
      clj-path output-path "cljs" cljs-default-transform-rules
      (constantly true)))

  ([clj-path output-path extension rules file-predicate]
     (doseq [f (find-clj-sources-in-dir (File. clj-path))]
       (when (file-predicate f)
         (let [generated-f (File. (-> (.getPath f)
                                      (string/replace clj-path output-path)
                                      (string/replace #"clj$" extension)))]
           (make-parents generated-f)
           (spit generated-f (transform-file f rules)))))))

(comment
  (transform-directory "src/dalap" ".generated")
  (print (string/join
          "\n"
          (cljs-transform (read-clj?s
                           "test/dalap/test/cljsrc_walk_sample.clj")))))

;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(deftest test-source-transformation
  (let [rules (concat
               ['something 'something-else
                'dalap.html.selector 'dalap.cljs.selector
                ] cljs-core-transform-rules)
        transform #(cljs-transform % rules)
        ]
    (is (= (transform '(something ^{:cljs something-else-cljs}
                                  foobar))
           '(something-else something-else-cljs)))

    (is (= (transform '(abcd ^:clj foobar))
           '(abcd)))

    (is (= (transform '(ns dalap.test.html.selector
                         (:use clojure.test)
                         (:use [clojure.pprint :only (pprint)])

                         (:require [dalap.html :as html])

                         ^{:cljs (:require [dalap.cljs.defaults :as defaults])}
                         (:require [dalap.defaults :as defaults])

                         (:use dalap.html.selector)
                         (:use [dalap.walk :only [walk]])
                         (:use [dalap.html :only [add-class]])))
           '(ns dalap.test.html.selector
              (:use clojure.test)
              (:use [clojure.pprint :only (pprint)])

              (:require [dalap.html :as html])
              (:require [dalap.cljs.defaults :as defaults])
              (:use dalap.cljs.selector)
              (:use [dalap.walk :only [walk]])
              (:use [dalap.html :only [add-class]]))))))

{'fraction_non_alphanumeric': 0.1562338334195551, 'fraction_numerical': 0.0001724435247456458, 'mean_word_length': 2.139956686518679, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1341336', 'n_tokens_mistral': 3953, 'n_tokens_neox': 3761, 'n_words': 925}