<commit_before>(aset js/aurora.language "operation" nil)

(ns aurora.language.operation
  (:require [clojure.set :refer [union intersection difference subset?]]
            [aurora.language.jsth :as jsth]
            [aurora.language.match :as match]
            [aurora.language.representation :refer [->Knowledge tick pretend-facts assert-facts retract-facts]])
  (:require-macros [aurora.macros :refer [console-time set!! conj!! disj!! assoc!!]]
                   [aurora.language.macros :refer [rule]]))

(def profile? false)

(let [next (atom 0)]
  (defn new-id []
    (if js/window.uuid
      (.replace (js/uuid) (js/RegExp. "-" "gi") "_")
      (swap! next inc))))

;; EXPRS

(defn expr->vars [expr]
  (cond
   (seq? expr) (apply union (map expr->vars (rest expr))) ;; first elem is function
   (coll? expr) (apply union (map expr->vars expr))
   (symbol? expr) #{expr}
   :else #{}))

(defn expr->jsth [expr]
  (clojure.walk/postwalk
   (fn [form]
     (if (and (seq? form) (symbol? (first form)) (not (.test #"\." (name (first form)))) (not= (namespace (first form)) "js") )
       (cons (symbol (str "cljs.core." (first form))) (rest form))
       form))
   expr))

(defn expr->fun [expr]
  (apply js/Function (conj (vec (expr->vars expr)) (jsth/statement->string `(return ~(expr->jsth expr))))))

;; MAPS -> ROWS

(defn select-ixes [vector ixes]
  (let [result #js []
        count (count ixes)]
    (loop [ix 0]
      (when (< ix count)
        (.push result (nth vector (nth ixes ix)))
        (recur (+ ix 1))))
    result))

(defn ix-of [vector value]
  (let [count (count vector)]
    (loop [ix 0]
      (if (< ix count)
        (if (= value (nth vector ix))
          ix
          (recur (+ ix 1)))
        (assert false (str (pr-str value) " is not contained in " (pr-str vector)))))))

(defn ixes-of [vector values]
  (vec (map #(ix-of vector %) values)))

;; PLAN NODES

(defprotocol PlanNode
  (run-node [this cache kn] "-> value"))

(defrecord Project [key pattern pattern-fn shape]
  PlanNode
  (run-node [this cache kn]
       (let [result (transient #{})
             facts (case key ;; TODO this is kind of hacky
                     :now&pretended (concat (:now kn) (:pretended kn))
                     :pretended (:pretended kn)
                     :asserted (:asserted kn)
                     :retracted (:retracted kn)
                     :now (:now kn))]
         (doseq [fact facts]
           (when-let [row (pattern-fn fact)]
             (conj!! result (js/cljs.core.PersistentVector.fromArray row true))))
         (persistent! result))))

(defn ->project [key pattern]
  (let [vars (match/vars pattern)
        shape (vec vars)
        pattern-fn (match/pattern pattern shape)]
    (->Project key pattern pattern-fn shape)))

;; (run-node (->project '[a b]) [] (->Knowledge nil nil nil #{[0] [1 2] [3 4 5]}))

(defrecord Join [i j key-ixes-i key-ixes-j select-ixes-i select-ixes-j shape]
  PlanNode
  (run-node [this cache kn]
       (let [result (transient #{})]
         (doseq [row-i (nth cache i)
                 :let [key-i (js/cljs.core.PersistentVector.fromArray (select-ixes row-i key-ixes-i) true)]
                 :let [_ (prn 'i key-i)]
                 row-j (nth cache j)
                 :let [key-j (js/cljs.core.PersistentVector.fromArray (select-ixes row-j key-ixes-j) true)]
                 :let [_ (prn 'j key-j)]
                 :when (= key-i key-j)]<commit_msg>I have had it with these motherfucking prns on this motherfucking plane<commit_after>(aset js/aurora.language "operation" nil)

(ns aurora.language.operation
  (:require [clojure.set :refer [union intersection difference subset?]]
            [aurora.language.jsth :as jsth]
            [aurora.language.match :as match]
            [aurora.language.representation :refer [->Knowledge tick pretend-facts assert-facts retract-facts]])
  (:require-macros [aurora.macros :refer [console-time set!! conj!! disj!! assoc!!]]
                   [aurora.language.macros :refer [rule]]))

(def profile? false)

(let [next (atom 0)]
  (defn new-id []
    (if js/window.uuid
      (.replace (js/uuid) (js/RegExp. "-" "gi") "_")
      (swap! next inc))))

;; EXPRS

(defn expr->vars [expr]
  (cond
   (seq? expr) (apply union (map expr->vars (rest expr))) ;; first elem is function
   (coll? expr) (apply union (map expr->vars expr))
   (symbol? expr) #{expr}
   :else #{}))

(defn expr->jsth [expr]
  (clojure.walk/postwalk
   (fn [form]
     (if (and (seq? form) (symbol? (first form)) (not (.test #"\." (name (first form)))) (not= (namespace (first form)) "js") )
       (cons (symbol (str "cljs.core." (first form))) (rest form))
       form))
   expr))

(defn expr->fun [expr]
  (apply js/Function (conj (vec (expr->vars expr)) (jsth/statement->string `(return ~(expr->jsth expr))))))

;; MAPS -> ROWS

(defn select-ixes [vector ixes]
  (let [result #js []
        count (count ixes)]
    (loop [ix 0]
      (when (< ix count)
        (.push result (nth vector (nth ixes ix)))
        (recur (+ ix 1))))
    result))

(defn ix-of [vector value]
  (let [count (count vector)]
    (loop [ix 0]
      (if (< ix count)
        (if (= value (nth vector ix))
          ix
          (recur (+ ix 1)))
        (assert false (str (pr-str value) " is not contained in " (pr-str vector)))))))

(defn ixes-of [vector values]
  (vec (map #(ix-of vector %) values)))

;; PLAN NODES

(defprotocol PlanNode
  (run-node [this cache kn] "-> value"))

(defrecord Project [key pattern pattern-fn shape]
  PlanNode
  (run-node [this cache kn]
       (let [result (transient #{})
             facts (case key ;; TODO this is kind of hacky
                     :now&pretended (concat (:now kn) (:pretended kn))
                     :pretended (:pretended kn)
                     :asserted (:asserted kn)
                     :retracted (:retracted kn)
                     :now (:now kn))]
         (doseq [fact facts]
           (when-let [row (pattern-fn fact)]
             (conj!! result (js/cljs.core.PersistentVector.fromArray row true))))
         (persistent! result))))

(defn ->project [key pattern]
  (let [vars (match/vars pattern)
        shape (vec vars)
        pattern-fn (match/pattern pattern shape)]
    (->Project key pattern pattern-fn shape)))

;; (run-node (->project '[a b]) [] (->Knowledge nil nil nil #{[0] [1 2] [3 4 5]}))

(defrecord Join [i j key-ixes-i key-ixes-j select-ixes-i select-ixes-j shape]
  PlanNode
  (run-node [this cache kn]
       (let [result (transient #{})]
         (doseq [row-i (nth cache i)
                 :let [key-i (js/cljs.core.PersistentVector.fromArray (select-ixes row-i key-ixes-i) true)]
                 row-j (nth cache j)
                 :let [key-j (js/cljs.core.PersistentVector.fromArray (select-ixes row-j key-ixes-j) true)]
                 :when (= key-i key-j)]
{'fraction_non_alphanumeric': 0.1627158326016974, 'fraction_numerical': 0.003219198127011999, 'mean_word_length': 2.383663366336634, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3727334', 'n_tokens_mistral': 2391, 'n_tokens_neox': 2284, 'n_words': 702}