<commit_before>(ns aurora.ast
  (:require [clojure.walk :refer [postwalk]]
            aurora.util)
  (:require-macros [aurora.macros :refer [check]]))

;; id = random 128bit string ;; either a notebook, page or step
;; js = string
;; ref = {:type :ref/step :step id} | {:type :ref/page :page id :notebook id} | {:type :ref/js :js js}
;; tag = {:type :tag :id id :name string} ;; like a namespaced keyword
;; call = {:type :call :ref ref :args [ref*]}
;; data = number | string | tag | [data*] | {data* data*} | ref
;; pattern = number | string | tag | [pattern*] | {data* pattern*} | ref ;; if ref is a step, check equality. if page, treat as predicate
;; action = call | data
;; branch = {:type :match/branch :pattern pattern :action action}
;; match = {:type :match :arg ref/step :branches [branch*]}
;; step = {:id id & (call | data | match)}
;; page = {:type :page :id id :steps [step*]}
;; notebook = {:type :notebook :id id :pages [page*]}

(defn id! [x]
  (check (string? x)))

(defn js! [x]
  (check (string? x)))

(defn ref-step! [x]
  (check (= :ref/step (:type x))
         (id! (:step x))))

(defn ref-page! [x]
  (check (= :ref/page (:type x))
         (id! (:page x))
         (id! (:notebook x))))

(defn ref-js! [x]
  (check (= :ref/js (:type x))
         (js! (:js x))))

(defn ref! [x]
  (case (:type x)
    :ref/step (ref-step! x)
    :ref/page (ref-page! x)
    :ref/js (ref-js! x)
    false))

(defn tag! [x]
  (check (= :tag (:type x))
         (id! (:id x))
         (string? (:name x))))

(defn call! [x]
  (check (= :call (:type x))
         (ref! (:ref x))
         (sequential? (:args x))
         (every? ref! (:args x))))

(defn data! [x]
  (check
   (cond
    (number? x) true
    (string? x) true
    (= :tag (:type x)) (tag! x)
    (vector? x) (every? data! x)
    (map? x) (and (every? data! (keys x)) (every? data! (vals x)))
    (#{:ref/js :ref/step :ref/page} (:type x)) (ref! x)
    :else false)))

(defn pattern! [x]
  (check
   (cond
    (number? x) true
    (string? x) true
    (= :tag (:type x)) (tag! x)
    (vector? x) (every? pattern! x)
    (map? x) (and (every? data! (keys x)) (every? pattern! (vals x)))
    (#{:ref/js :ref/step :ref/page} (:type x)) (ref! x)
    :else false)))

(defn action! [x]
  (check
   (case (:type x)
     :call (call! x)
     :data (data! x)
     false)))

(defn branch! [x]
  (check (= :match/branch (:type x))
         (pattern! (:pattern x))
         (action! (:action x))))

(defn match! [x]
  (check (= :match (:type x))
         (ref-step! (:arg match))
         (sequential? (:branches x))
         (every! branch! (:branches x))))

(defn step! [x]
  (check (id! (:id x))
         (case (:type x)
           :call (call! x)
           :data (data! x)
           :match (match! x))))

(defn page! [x]
  (check (= :page (:type x))
         (id! (:id x))
         (sequential? (:steps x))
         (every? step! (:steps x))))

(defn notebook [x]
  (check (= :notebook (:type x))
         (id! (:id x))
         (sequential? (:pages x))
         (every? page! (:pages x))))
<commit_msg>Tweak ast and translate example-a<commit_after>(ns aurora.ast
  (:require [clojure.walk :refer [postwalk]]
            aurora.util)
  (:require-macros [aurora.macros :refer [check]]))

(defn id! [x]
  (check (string? x)))

(defn js! [x]
  (check (string? x)))

(defn ref-id! [x]
  (check (= :ref/id (:type x))
         (id! (:id x))))

(defn ref-js! [x]
  (check (= :ref/js (:type x))
         (js! (:js x))))

(defn ref! [x]
  (case (:type x)
    :ref/id (ref-id! x)
    :ref/js (ref-js! x)
    false))

(defn tag! [x]
  (check (= :tag (:type x))
         (id! (:id x))
         (string? (:name x))))

(defn call! [x]
  (check (= :call (:type x))
         (ref! (:ref x))
         (sequential? (:args x))
         (every? ref! (:args x))))

(defn data! [x]
  (check
   (cond
    (number? x) true
    (string? x) true
    (= :tag (:type x)) (tag! x)
    (vector? x) (every? data! x)
    (map? x) (and (every? data! (keys x)) (every? data! (vals x)))
    (#{:ref/js :ref/step :ref/page} (:type x)) (ref! x)
    :else false)))

(defn constant! [x]
  (check (= :constant (:type x))
         (data! (:data x))))

(defn pattern! [x]
  (check
   (cond
    (number? x) true
    (string? x) true
    (= :tag (:type x)) (tag! x)
    (vector? x) (every? pattern! x)
    (map? x) (and (every? data! (keys x)) (every? pattern! (vals x)))
    (#{:ref/id :ref/page} (:type x)) (ref! x)
    :else false)))

(defn pattern! [x]
  (check (= :type )))

(defn action! [x]
  (check
   (case (:type x)
     :call (call! x)
     :constant (constant! x)
     false)))

(defn branch! [x]
  (check (= :match/branch (:type x))
         (pattern! (:pattern x))
         (action! (:action x))))

(defn match! [x]
  (check (= :match (:type x))
         (ref-step! (:arg match))
         (sequential? (:branches x))
         (every! branch! (:branches x))))

(defn step! [x]
  (check (id! (:id x))
         (case (:type x)
           :call (call! x)
           :constant (constant! x)
           :match (match! x))))

(defn page! [x]
  (check (= :page (:type x))
         (id! (:id x))
         (sequential? (:args x))
         (every? id! (:args x))
         (sequential? (:steps x))
         (every? step! (:steps x))))

(defn notebook! [x]
  (check (= :notebook (:type x))
         (id! (:id x))
         (sequential? (:pages x))
         (every? page! (:pages x))))

;; examples

(def example-a
  {:type :notebook
   :id "example_a"
   :pages [{:type :page
            :id "root"
            :args ["a" "b" "c"]
            :steps [{:id "b_squared"
                     :type :call
                     :ref {:type :ref/js
                           :js "cljs.core._STAR_"}
                     :args [{:type :ref/id
                             :id "b"}
                            {:type :ref/id
                             :id "b"}]}
                    {:id "four"
                     :type :constant
                     :data 4}
                    {:id "four_a_c"
                     :type :call
                     :ref {:type :ref/js
                           :js "cljs.core._STAR_"}
                     :args [{:type :ref/id
                             :id "four"}
                            {:type :ref/id
                             :id "a"}
                            {:type :ref/id
                             :id "c"}]}
                    {:id "result"
                     :type :call
                     :ref {:type :ref/js
                           :js "cljs.core._"}
                     :args [{:type :ref/id
                             :id "b_squared"}
                            {:type :ref/id
                             :id "four_a_c"}]}]}]})

(notebook! example-a)
{'fraction_non_alphanumeric': 0.2013641755634638, 'fraction_numerical': 0.0005931198102016608, 'mean_word_length': 1.7975943591870593, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1958560', 'n_tokens_mistral': 2557, 'n_tokens_neox': 2420, 'n_words': 764}