<commit_before>(ns refac.core
  (:gen-class)
  (use refac.io)
  (use refac.java))

(defn ignore-to-eol [[first & others]]
  (case first
    ";" others
    (recur others)))

(defn do-package [items] [(ignore-to-eol items) {}])
(defn do-import  [items] [(ignore-to-eol items) {}])

(defn do-open-body [[first & others]]
  (if-not (= first "{")
    (throw (IllegalStateException. (str "Expected {, got " first)))
    others))

(defn do-body [items]
  (loop [[first & others] (do-open-body items)
         indent 1]
    (println first)
    (when-not (= indent 0)
      (recur
        others
         (+ indent
            (case first
              "{"  1
              "}" -1
              0))
        ))))

(defn do-type [[first second & others]]
  (let [info {:type first :name second}
        remaining (case first
                     "class" (do-body others))]
    [remaining info]))

(defn do-declaration [items]
  (let [[first & others] items
        [visability [remaining info]]
          (case first
            ("public" "private" "protected" "internal") [first (do-type others)]
            ["private" (do-type items)])]
    [remaining (assoc info :visability visability)]))

(defn do-root [items]
  (loop [[items _] [items {}]]
    (let [[first & others] items]
      (println first)
      (recur
        (case first
          "package" (do-package others)
          "import"  (do-import others)
          (do-declaration items))))))

(defn process-symbols [symbols]
  (do-root symbols))


(defn -main
  [& args]
  (process-symbols
    (get-symbol-stream)))


<commit_msg>Adding method ignore.<commit_after>(ns refac.core
  (:gen-class)
  (use refac.io)
  (use refac.java)
  (:require [clojure.string :as s]))

(defn ignore-to [[first & others] stop-at]
  {:pre [(not (nil? first))]}
  (if (= first stop-at) others
    (recur others stop-at)))

(defn nested-ignore-to [open close items]
  (loop [items items
         indent 1]

    (let [[next & others] items
          indent (+ indent
                    (condp = next
                      open   1
                      close -1
                      0))]
      (if (= indent 0)
        others
        (do
          (if (nil? items) (throw (IllegalArgumentException. "Did not find closing.")))
          (recur others indent))))))

(defn ignore-to-eol [items] (ignore-to items ";"))

(defn do-package [items] [(ignore-to-eol items) {}])
(defn do-import  [items] [(ignore-to-eol items) {}])

(defn do-method [items]
  (->> (ignore-to items "{")
       (nested-ignore-to "{" "}")))

(defn do-assignment [items] (ignore-to-eol items))

(declare do-declaration)

(defn do-body [items]
  (let [[next & others] items]
    (case next
      "}" others
      (recur
        (let [[remaining _] (do-declaration items)]
          remaining)))))

(defn do-declaration-name [[name assignment & others]]
  (let [info {:name name}
        remaining (case assignment
                    "{" (do-body others)
                    "(" (do-method others)
                    "=" (do-assignment others)
                    ";" others)]
    [remaining info]))

(defn do-generic-type [items]
  (loop [items items
         depth 1
         generic "<"]
    (let [[first & others] items]
      (if-not (= depth 0)
        (recur
          others
          (+ depth
             (case first
               "<"  1
               ">" -1
               0))
          (str generic first))
        [items generic]))))

(defn do-declaration-type [items]
  (let [[type generic-declaration & others] items
        [remaining generic] (case generic-declaration
                              "<" (do-generic-type others)
                              nil)
        has-generic (not (nil? generic))
        remaining (if has-generic
                    remaining
                    (cons generic-declaration others))
        type {:type (str type generic)}

        [remaining info] (do-declaration-name remaining)]
    [remaining (merge info type)]))

(defn log [[remaining info]]
  (println info)
  [remaining info])

(defn do-declaration [items]
  (let [[first & others] items
        declaration (case first
                      ("public" "private" "protected" "internal") [first (do-declaration-type others)]
                      ["private" (do-declaration-type items)])
        [visability [remaining info]] declaration]

    (log [remaining (merge info {:visability visability})])))

(defn do-root [items]
  (loop [[items _] [items {}]]
    (let [[next & others] items]
      (recur
        (case next
          "package" (do-package others)
          "import"  (do-import others)
          (do-declaration items))))))

(defn process-symbols [symbols]
  (do-root symbols))

(defn -main [] (process-symbols (get-symbol-stream)))
{'fraction_non_alphanumeric': 0.1505669886602268, 'fraction_numerical': 0.003149937001259975, 'mean_word_length': 1.976875, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5351923', 'n_tokens_mistral': 1519, 'n_tokens_neox': 1479, 'n_words': 411}