<commit_before>;; Copyright (c) <NAME>, 2012. All rights reserved. The use and
;; distribution terms for this software are covered by the Eclipse
;; Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
;; which can be found in the file epl-v10.html at the root of this
;; distribution. By using this software in any fashion, you are
;; agreeing to be bound by the terms of this license. You must not
;; remove this notice, or any other, from this software.

(ns ^{:author "<NAME>"
      :doc "Refactoring tool to move a Clojure namespace from one name/file to
  another, and update all references to that namespace in your other
  Clojure source files.

  WARNING: This code is ALPHA and subject to change. It also modifies
  and deletes your source files! Make sure you have a backup or
  version control.

  DISCLAIMER
  This is patched version of <NAME>'s original to handle cljc files

"}
  mranderson.move
  (:require [clojure.string :as str]
            [clojure.java.io :as io]
            [mranderson.util :as util]
            [parallel.core :as p]
            [rewrite-clj.zip :as z]
            [rewrite-clj.zip.base :as b]
            [rewrite-clj.parser :as parser]
            [rewrite-clj.reader :as reader])
  (:import (java.io File FileNotFoundException PushbackReader)))

(defn- update-file
  "Reads file as a string, calls f on the string plus any args, then
  writes out return value of f as the new contents of file. Does not
  modify file if the content is unchanged."
  [file f & args]
  (let [old (slurp file)
        new (str (apply f old args))]
    (when-not (= old new)
      (spit file new))))

(defn- sym->file
  [path sym extension]
  (io/file path (str (util/sym->file-name sym) extension)))

(defn- update? [file extension-of-moved]
  (let [file-ext (util/file->extension file)
        all-extensions #{".cljc" ".cljs" ".clj"}]
    (or
     (and (= ".cljc" extension-of-moved)
          (all-extensions file-ext))
     (= file-ext extension-of-moved)
     (= file-ext ".cljc"))))

(defn- clojure-source-files [dirs extension]
  (->> dirs
       (map io/file)
       (filter #(.exists ^File %))
       (mapcat file-seq)
       (filter (fn [^File file]
                 (and (.isFile file)
                      (update? (str file) extension))))
       (map #(.getCanonicalFile ^File %))))

(defn- prefix-libspec [libspec]
  (let [prefix (str/join "." (butlast (str/split (name libspec) #"\.")))]
    (and prefix (symbol prefix))))

(defn- java-package [sym]
  (str/replace (name sym) "-" "_"))

(defn- sexpr=
  [node-sexpr old-sym]
  (= node-sexpr old-sym))

(defn- java-style-prefix?
  [node node-sexpr old-sym]
  (let [old-as-java-pkg (java-package old-sym)
        java-pkg-prefix (str old-as-java-pkg ".")]
    (and
     (str/includes? (name old-sym) "-")
     (or (= node-sexpr (symbol old-as-java-pkg))
         (str/starts-with? node-sexpr java-pkg-prefix)))))

(defn- libspec-prefix?
  [node node-sexpr old-sym]
  (let [old-sym-prefix-libspec (prefix-libspec old-sym)
        parent-leftmost-node  (z/leftmost (z/up node))
        parent-leftmost-sexpr (and parent-leftmost-node
                                   (not
                                    (#{:uneval}
                                     (b/tag parent-leftmost-node)))
                                   (b/sexpr parent-leftmost-node))]
    (and (= :require parent-leftmost-sexpr)
         (= node-sexpr old-sym-prefix-libspec))))

(defn- contains-sym? [old-sym node]
  (when-not (#{:uneval} (b/tag node))
    (when-let [node-sexpr (b/sexpr node)]
      (or
       (sexpr= node-sexpr old-sym)
       (java-style-prefix? node node-sexpr old-sym)
       (libspec-prefix? node node-sexpr old-sym)))))

(defn- ->new-node [old-node old-sym new-sym]
  (let [old-prefix (prefix-libspec old-sym)]
    (cond-> old-node

      :always
      (str/replace-first
       (name old-sym)
       (name new-sym))

      (str/includes? (name old-sym) "-")
      (str/replace-first (java-package old-sym) (java-package new-sym))

      (= old-prefix old-node)
      (str/replace-first
       (name old-prefix)
       (name (prefix-libspec new-sym))))))

(defn- replace-in-node [old-sym new-sym old-node]
  (let [new-node (->new-node old-node old-sym new-sym)]
    (cond
      (symbol? old-node) (symbol new-node)

      :default new-node)))

(defn- ns-decl? [node]
  (when-not (#{:uneval} (b/tag node))
    (= 'ns (b/sexpr (z/down node)))))

(def ^:const ns-form-placeholder (str "ns_" "form_" "placeholder"))

(defn- split-ns-form-ns-body
  "Returns ns form as a rewrite-clj loc and the ns body as string with a place holder for the ns form."
  [content]
  (let [reader     (reader/string-reader content)
        first-form (parser/parse reader)]
    (loop [ns-form-maybe (and first-form (z/edn first-form))]
      (if (ns-decl? ns-form-maybe)
        [ns-form-maybe
         (str/replace content (z/root-string ns-form-maybe) ns-form-placeholder)]
        (if-let [next-form (parser/parse reader)]
          (recur (z/edn next-form))
          [nil content])))))

(defn- replace-in-ns-form [ns-loc old-sym new-sym]
  (loop [loc ns-loc]
    (if-let [found-node (some-> (z/find-next-depth-first loc (partial contains-sym? old-sym))
                                (z/edit (partial replace-in-node old-sym new-sym)))]
      (recur found-node)
      (z/root-string loc))))

(defn- source-replacement [old-sym new-sym match]
  (let [old-ns-ref      (name old-sym)
        new-ns-ref      (name new-sym)
        old-pkg-prefix  (java-package old-sym)
        new-pkg-prefix  (java-package new-sym)
        old-type-prefix (str "^" (java-package old-sym))
        new-type-prefix (str "^" (java-package new-sym))]
    (cond

      (= match old-ns-ref)
      new-ns-ref

      (and (str/starts-with? match old-pkg-prefix)
           (str/includes? match "_"))
      (str/replace match old-pkg-prefix new-pkg-prefix)

      (str/starts-with? match old-type-prefix)
      (str/replace match old-type-prefix new-type-prefix)

      :default
      match)))

(def ^:private symbol-regex
  ;; LispReader.java uses #"[:]?([\D&&[^/]].*/)?([\D&&[^/]][^/]*)" but
  ;; that's too broad; we don't want a whole namespace-qualified symbol,
  ;; just each part individually.
  #"\"?[a-zA-Z0-9$%*+=?!<>^_-]['.a-zA-Z0-9$%*+=?!<>_-]*")

(defn- replace-in-source [source-sans-ns old-sym new-sym]
  (str/replace source-sans-ns symbol-regex (partial source-replacement old-sym new-sym)))

(defn- after-platfrom-marker? [platform node]
  (when-not (#{:uneval} (b/tag node))
    (= platform (b/sexpr (z/left node)))))

(defn- find-and-replace-platform-specific-subforms [platform ns-loc]
  (loop [loc         ns-loc
         found-nodes []]
    (if-let [found-node (z/find-next-depth-first loc (partial after-platfrom-marker? platform))]
      (recur (z/replace found-node (symbol (str (name platform) "_require"))) (conj found-nodes found-node))
      [found-nodes (z/of-string (z/root-string loc))])))

(defn- restore-platform-specific-subforms [platform replaced-nodes ns-form]
  (loop [form             ns-form
         [n & rest-nodes] replaced-nodes]
    (if-not n
      form
      (recur (str/replace-first form (str (name platform) "_require") (z/string n)) rest-nodes))))

(defn replace-ns-symbol
  "ALPHA: subject to change. Given Clojure source as a file, replaces
  all occurrences of the namespace name old-sym with new-sym and
  returns modified source as a string.

  Splits the source file, parses the ns macro if found to do all the necessary
  transformations. Works on the body of namepsace as text as simpler transformations
  are needed. When done puts the ns form and body back together."
  [content old-sym new-sym extension-of-moved file-ext]
  (let [[ns-loc source-sans-ns] (split-ns-form-ns-body content)
        opp-platform       (util/platform-comp (util/extension->platform extension-of-moved))
        [replaced-nodes ns-loc] (or (and (= ".cljc" file-ext) opp-platform
                                         (find-and-replace-platform-specific-subforms opp-platform ns-loc))
                                    [[] ns-loc])

        new-ns-form             (future (replace-in-ns-form ns-loc old-sym new-sym))
        new-source-sans-ns      (future (replace-in-source source-sans-ns old-sym new-sym))
        new-ns-form             (if (seq replaced-nodes)
                                  (restore-platform-specific-subforms opp-platform replaced-nodes @new-ns-form)<commit_msg>[refactor] Simplify getting back to a root zipper<commit_after>;; Copyright (c) <NAME>, 2012. All rights reserved. The use and
;; distribution terms for this software are covered by the Eclipse
;; Public License 1.0 (http://opensource.org/licenses/eclipse-1.0.php)
;; which can be found in the file epl-v10.html at the root of this
;; distribution. By using this software in any fashion, you are
;; agreeing to be bound by the terms of this license. You must not
;; remove this notice, or any other, from this software.

(ns ^{:author "<NAME>"
      :doc "Refactoring tool to move a Clojure namespace from one name/file to
  another, and update all references to that namespace in your other
  Clojure source files.

  WARNING: This code is ALPHA and subject to change. It also modifies
  and deletes your source files! Make sure you have a backup or
  version control.

  DISCLAIMER
  This is patched version of Stuart Siearra's original to handle cljc files

"}
  mranderson.move
  (:require [clojure.string :as str]
            [clojure.java.io :as io]
            [mranderson.util :as util]
            [parallel.core :as p]
            [rewrite-clj.zip :as z]
            [rewrite-clj.zip.base :as b]
            [rewrite-clj.parser :as parser]
            [rewrite-clj.reader :as reader])
  (:import (java.io File FileNotFoundException PushbackReader)))

(defn- update-file
  "Reads file as a string, calls f on the string plus any args, then
  writes out return value of f as the new contents of file. Does not
  modify file if the content is unchanged."
  [file f & args]
  (let [old (slurp file)
        new (str (apply f old args))]
    (when-not (= old new)
      (spit file new))))

(defn- sym->file
  [path sym extension]
  (io/file path (str (util/sym->file-name sym) extension)))

(defn- update? [file extension-of-moved]
  (let [file-ext (util/file->extension file)
        all-extensions #{".cljc" ".cljs" ".clj"}]
    (or
     (and (= ".cljc" extension-of-moved)
          (all-extensions file-ext))
     (= file-ext extension-of-moved)
     (= file-ext ".cljc"))))

(defn- clojure-source-files [dirs extension]
  (->> dirs
       (map io/file)
       (filter #(.exists ^File %))
       (mapcat file-seq)
       (filter (fn [^File file]
                 (and (.isFile file)
                      (update? (str file) extension))))
       (map #(.getCanonicalFile ^File %))))

(defn- prefix-libspec [libspec]
  (let [prefix (str/join "." (butlast (str/split (name libspec) #"\.")))]
    (and prefix (symbol prefix))))

(defn- java-package [sym]
  (str/replace (name sym) "-" "_"))

(defn- sexpr=
  [node-sexpr old-sym]
  (= node-sexpr old-sym))

(defn- java-style-prefix?
  [node node-sexpr old-sym]
  (let [old-as-java-pkg (java-package old-sym)
        java-pkg-prefix (str old-as-java-pkg ".")]
    (and
     (str/includes? (name old-sym) "-")
     (or (= node-sexpr (symbol old-as-java-pkg))
         (str/starts-with? node-sexpr java-pkg-prefix)))))

(defn- libspec-prefix?
  [node node-sexpr old-sym]
  (let [old-sym-prefix-libspec (prefix-libspec old-sym)
        parent-leftmost-node  (z/leftmost (z/up node))
        parent-leftmost-sexpr (and parent-leftmost-node
                                   (not
                                    (#{:uneval}
                                     (b/tag parent-leftmost-node)))
                                   (b/sexpr parent-leftmost-node))]
    (and (= :require parent-leftmost-sexpr)
         (= node-sexpr old-sym-prefix-libspec))))

(defn- contains-sym? [old-sym node]
  (when-not (#{:uneval} (b/tag node))
    (when-let [node-sexpr (b/sexpr node)]
      (or
       (sexpr= node-sexpr old-sym)
       (java-style-prefix? node node-sexpr old-sym)
       (libspec-prefix? node node-sexpr old-sym)))))

(defn- ->new-node [old-node old-sym new-sym]
  (let [old-prefix (prefix-libspec old-sym)]
    (cond-> old-node

      :always
      (str/replace-first
       (name old-sym)
       (name new-sym))

      (str/includes? (name old-sym) "-")
      (str/replace-first (java-package old-sym) (java-package new-sym))

      (= old-prefix old-node)
      (str/replace-first
       (name old-prefix)
       (name (prefix-libspec new-sym))))))

(defn- replace-in-node [old-sym new-sym old-node]
  (let [new-node (->new-node old-node old-sym new-sym)]
    (cond
      (symbol? old-node) (symbol new-node)

      :default new-node)))

(defn- ns-decl? [node]
  (when-not (#{:uneval} (b/tag node))
    (= 'ns (b/sexpr (z/down node)))))

(def ^:const ns-form-placeholder (str "ns_" "form_" "placeholder"))

(defn- split-ns-form-ns-body
  "Returns ns form as a rewrite-clj loc and the ns body as string with a place holder for the ns form."
  [content]
  (let [reader     (reader/string-reader content)
        first-form (parser/parse reader)]
    (loop [ns-form-maybe (and first-form (z/edn first-form))]
      (if (ns-decl? ns-form-maybe)
        [ns-form-maybe
         (str/replace content (z/root-string ns-form-maybe) ns-form-placeholder)]
        (if-let [next-form (parser/parse reader)]
          (recur (z/edn next-form))
          [nil content])))))

(defn- replace-in-ns-form [ns-loc old-sym new-sym]
  (loop [loc ns-loc]
    (if-let [found-node (some-> (z/find-next-depth-first loc (partial contains-sym? old-sym))
                                (z/edit (partial replace-in-node old-sym new-sym)))]
      (recur found-node)
      (z/root-string loc))))

(defn- source-replacement [old-sym new-sym match]
  (let [old-ns-ref      (name old-sym)
        new-ns-ref      (name new-sym)
        old-pkg-prefix  (java-package old-sym)
        new-pkg-prefix  (java-package new-sym)
        old-type-prefix (str "^" (java-package old-sym))
        new-type-prefix (str "^" (java-package new-sym))]
    (cond

      (= match old-ns-ref)
      new-ns-ref

      (and (str/starts-with? match old-pkg-prefix)
           (str/includes? match "_"))
      (str/replace match old-pkg-prefix new-pkg-prefix)

      (str/starts-with? match old-type-prefix)
      (str/replace match old-type-prefix new-type-prefix)

      :default
      match)))

(def ^:private symbol-regex
  ;; LispReader.java uses #"[:]?([\D&&[^/]].*/)?([\D&&[^/]][^/]*)" but
  ;; that's too broad; we don't want a whole namespace-qualified symbol,
  ;; just each part individually.
  #"\"?[a-zA-Z0-9$%*+=?!<>^_-]['.a-zA-Z0-9$%*+=?!<>_-]*")

(defn- replace-in-source [source-sans-ns old-sym new-sym]
  (str/replace source-sans-ns symbol-regex (partial source-replacement old-sym new-sym)))

(defn- after-platfrom-marker? [platform node]
  (when-not (#{:uneval} (b/tag node))
    (= platform (b/sexpr (z/left node)))))

(defn- find-and-replace-platform-specific-subforms [platform ns-loc]
  (loop [loc         ns-loc
         found-nodes []]
    (if-let [found-node (z/find-next-depth-first loc (partial after-platfrom-marker? platform))]
      (recur (z/replace found-node (symbol (str (name platform) "_require"))) (conj found-nodes found-node))
      [found-nodes (z/edn (z/root loc))])))

(defn- restore-platform-specific-subforms [platform replaced-nodes ns-form]
  (loop [form             ns-form
         [n & rest-nodes] replaced-nodes]
    (if-not n
      form
      (recur (str/replace-first form (str (name platform) "_require") (z/string n)) rest-nodes))))

(defn replace-ns-symbol
  "ALPHA: subject to change. Given Clojure source as a file, replaces
  all occurrences of the namespace name old-sym with new-sym and
  returns modified source as a string.

  Splits the source file, parses the ns macro if found to do all the necessary
  transformations. Works on the body of namepsace as text as simpler transformations
  are needed. When done puts the ns form and body back together."
  [content old-sym new-sym extension-of-moved file-ext]
  (let [[ns-loc source-sans-ns] (split-ns-form-ns-body content)
        opp-platform       (util/platform-comp (util/extension->platform extension-of-moved))
        [replaced-nodes ns-loc] (or (and (= ".cljc" file-ext) opp-platform
                                         (find-and-replace-platform-specific-subforms opp-platform ns-loc))
                                    [[] ns-loc])

        new-ns-form             (future (replace-in-ns-form ns-loc old-sym new-sym))
        new-source-sans-ns      (future (replace-in-source source-sans-ns old-sym new-sym))
        new-ns-form             (if (seq replaced-nodes)
                                  (restore-platform-specific-subforms opp-platform replaced-nodes @new-ns-form)
{'fraction_non_alphanumeric': 0.14913260805645398, 'fraction_numerical': 0.0016465745369009116, 'mean_word_length': 2.414173860670548, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4192083', 'n_tokens_mistral': 5813, 'n_tokens_neox': 5654, 'n_words': 1595}