<commit_before>(ns compound2.core)

#?(:cljs (enable-console-print!))

(defprotocol Index
  (id [this])
  (extract-key [this x])
  (index [this coll k x])
  (unindex [this coll k x])
  (before [this coll])
  (after [this coll]))

(defprotocol PrimaryIndex
  (get-by-key [this coll k])
  (get-all [this coll])
  (on-conflict [this old new]))

(defprotocol Compound
  :extend-via-metadata true
  (items [c])
  (add-items [c xs])
  (remove-keys [c ks]))

(defmulti indexer :index-type)

(defmethod indexer :one-to-one
  [opts]
  (let [{:keys [id kfn]} opts
        id (or id (when (keyword? kfn) kfn))]
    (assert (some? id) "Must provide an id")
    (reify
      PrimaryIndex
      (get-by-key [this coll k]
        (get coll k))
      (get-all [this coll]
        (vals coll))
      (on-conflict [this old new]
        new)
      Index
      (id [this]
        id)
      (extract-key [this x]
        (kfn x))
      (index [this coll k x]
        (assoc! coll k x))
      (unindex [this coll k x]
        (dissoc! coll k))
      (after [this coll]
        (persistent! coll))
      (before [this coll]
        (transient (or coll {}))))))

(defmethod indexer :one-to-many
  [opts]
  (let [{:keys [id kfn]} opts
        id (or id (when (keyword? kfn) kfn))]
    (assert (some? id) "Must provide an id")
    (reify
      Index
      (id [this]
        id)
      (extract-key [this x]
        (kfn x))
      (index [this coll k x]
        (let [existing (get coll k)]
          (assoc! coll k (conj (or existing #{}) x))))
      (unindex [this coll k x]
        (let [existing (get coll k)
              new (disj existing x)]
          (if (seq new)
            (assoc! coll k new)
            (dissoc! coll k))))
      (after [this coll]
        (persistent! coll))
      (before [this coll]
        (transient (or coll {}))))))

(defn dissoc-in
  [m [k & ks]]
  (if ks
    (let [m2 (dissoc-in (m k) ks)]
      (if (empty? m2)
        (dissoc m k)
        (assoc m k m2)))
    (dissoc m k)))

(defmethod indexer :nested-to-one
  [opts]
  (let [{:keys [id path]} opts
        id (or id (when (vector? path) path))]
    (assert (some? id) "Must provide an id")
    (reify
      Index
      (id [this]
        id)
      (extract-key [this x]
        (into [] (for [p path]
                   (p x))))
      (index [this coll path x]
        (assoc-in coll path x))
      (unindex [this coll path x]
        (dissoc-in coll path))
      (after [this coll]
        coll)
      (before [this coll]
        (or coll {})))))

(defmethod indexer :nested-to-many
  [opts]
  (let [{:keys [id path]} opts
        id (or id (when (vector? path) path))]
    (assert (some? id) "Must provide an id")
    (reify
      Index
      (id [this]
        id)
      (extract-key [this x]
        (into [] (for [p path]
                   (p x))))
      (index [this coll path x]
        (update-in coll path (fnil conj #{})))
      (unindex [this coll path x]
        (let [existing (get-in coll path)
              new (disj existing x)]
          (if (seq new)
            (assoc-in coll path new)
            (update-in coll (butlast path) dissoc (last path)))))
      (after [this coll]
        coll)
      (before [this coll]
        (or coll {})))))

(defmethod indexer :many-to-many
  [opts]
  (let [{:keys [id kfn]} opts
        id (or id (when (keyword? kfn) kfn))]
    (assert (some? id) "Must provide an id")
    (reify
      Index
      (id [this]
        id)
      (extract-key [this x]
        (kfn x))
      (index [this coll ks x]
        (reduce
         (fn [acc k]
           (assoc! acc k x))
         coll
         ks))
      (unindex [this coll ks x]
        (reduce
         (fn [acc k]
           (dissoc! acc k x))
         coll
         ks))
      (after [this coll]
        (persistent! coll))
      (before [this coll]
        (transient (or coll {}))))))

#?(:clj
   (defmacro compound [indexes]
     (let [[p-opts & s-opts] indexes
           pt-sym (gensym "pt")
           pi-sym (gensym "pi")
           st-syms (for [_ s-opts]
                     (gensym "st"))
           si-syms (for [_ s-opts]
                     (gensym "si"))
           m-sym (gensym "m")
           x-sym (gensym "x")
           k-sym (gensym "k")
           ex-sym (gensym "ex")]
       `(let [~pt-sym (indexer ~p-opts)
              ~@(mapcat (fn [sym opts]
                          [sym `(indexer ~opts)])
                        st-syms
                        s-opts)]
          (assert (satisfies? PrimaryIndex ~pt-sym) "First index must be a primary index")
          (with-meta {}
            {`items (fn [~m-sym]
                      (get-all ~pt-sym (get ~m-sym (id ~pt-sym))))
             `add-items (fn [~m-sym xs#]
                          (loop [~pi-sym (before ~pt-sym (get ~m-sym (id ~pt-sym)))
                                 ~@(mapcat (fn [st si]
                                             [si `(before ~st (get ~m-sym (id ~st)))])
                                           st-syms
                                           si-syms)
                                 [~x-sym & xs#] xs#]
                            (if (nil? ~x-sym)
                              (with-meta ~(into {`(id ~pt-sym) `(after ~pt-sym ~pi-sym)}
                                                (map (fn [st si]
                                                       [`(id ~st)
                                                        `(after ~st ~si)])
                                                     st-syms
                                                     si-syms))
                                (meta ~m-sym))
                              (let [k# (extract-key ~pt-sym ~x-sym)
                                    ~ex-sym (get-by-key ~pt-sym ~pi-sym k#)]
                                (if ~ex-sym
                                  (recur
                                   (index ~pt-sym ~pi-sym k# ~x-sym)
                                   ~@(map (fn [st si]
                                            `(let [k1# (extract-key ~st ~ex-sym)
                                                   k2# (extract-key ~st ~x-sym)]
                                               (index ~st (unindex ~st ~si k1# ~ex-sym) k2# ~x-sym)))
                                          st-syms
                                          si-syms)
                                   xs#)
                                  (recur
                                   (index ~pt-sym ~pi-sym k# ~x-sym)
                                   ~@(map (fn [st si]
                                            `(let [k# (extract-key ~st ~x-sym)]
                                               (index ~st ~si k# ~x-sym)))
                                          st-syms
                                          si-syms)
                                   xs#))))))
             `remove-keys (fn [~m-sym ks#]
                            (loop [~pi-sym (before ~pt-sym (get ~m-sym (id ~pt-sym)))
                                   ~@(mapcat (fn [st si]
                                               [si `(before ~st (get ~m-sym (id ~st)))])
                                             st-syms
                                             si-syms)
                                   [~k-sym & ks#] ks#]
                              (if (nil? ~k-sym)
                                (with-meta ~(into {`(id ~pt-sym) `(after ~pt-sym ~pi-sym)}
                                                  (map (fn [st si]
                                                         [`(id ~st)
                                                          `(after ~st ~si)])
                                                       st-syms
                                                       si-syms))
                                  (meta ~m-sym))
                                (let [~ex-sym (get-by-key ~pt-sym ~pi-sym ~k-sym)]
                                  (if ~ex-sym
                                    (recur
                                     (unindex ~pt-sym ~pi-sym ~k-sym ~ex-sym)
                                     ~@(map (fn [st si]
                                              `(let [k# (extract-key ~st ~ex-sym)]
                                                 (unindex ~st ~si k# ~ex-sym)))
                                            st-syms
                                            si-syms)
                                     ks#)
                                    (recur
                                     ~pi-sym
                                     ~@si-syms
                                     ks#))))))})))))

(defn compound* [indexes]
  (let [[pi & sis] (doall (map indexer indexes))]
    (assert (satisfies? PrimaryIndex pi) "First index must be a primary index")
    (with-meta {}
      {`items (fn [c]
                (get-all pi (get c (id pi))))
       `add-items (fn [c xs]
                    (loop [px (before pi (get c (id pi)))
                           sixs (reduce (fn [acc si]
                                          (assoc acc si (before si (get c (id si)))))
                                        {}
                                        sis)
                           [x & xs] xs]
                      (if (nil? x)
                        (with-meta
                          (reduce-kv (fn [acc si sx]
                                       (assoc acc (id si) (after si sx)))
                                     {(id pi) (after pi px)}
                                     sixs)
                          (meta c))
                        (let [k (extract-key pi x)
                              ex (get-by-key pi px k)]
                          (if ex
                            (recur (index pi px k (on-conflict pi ex x))
                                   (reduce-kv (fn [acc si sx]
                                                (assoc acc si (let [k1 (extract-key si ex)
                                                                    k2 (extract-key si x)]
                                                                (index si (unindex si sx k1 ex) k2 x))))
                                              {}
                                              sixs)
                                   xs)
                            (recur (index pi px k x)
                                   (reduce-kv (fn [acc si sx]
                                                (assoc acc si (let [k (extract-key si x)]
                                                                (index si sx k x))))
                                              {}
                                              sixs)
                                   xs))))))
       `remove-keys (fn [c ks]
                      (loop [px (before pi (get c (id pi)))
                             sixs (reduce (fn [acc si]
                                            (assoc acc si (before si (get c (id si)))))
                                          {}
                                          sis)
                             [k & ks] ks]
                        (if (nil? k)
                          (with-meta
                            (reduce-kv (fn [acc si sx]
                                         (assoc acc (id si) (after si sx)))
                                       {(id pi) (after pi px)}
                                       sixs)
                            (meta c)))
                        (if-let [ex (get-by-key pi px k)]
                          (recur (unindex pi px k ex)
                                 (reduce-kv (fn [acc si sx]
                                              (assoc acc si (let [k (extract-key si ex)]
                                                              (unindex si sx k ex))))
                                            {}
                                            sixs)
                                 ks)
                          (recur px sixs ks))))})))<commit_msg>Fix transient used after persistent! call<commit_after>(ns compound2.core)

#?(:cljs (enable-console-print!))

(defprotocol Index
  (id [this])
  (extract-key [this x])
  (index [this coll k x])
  (unindex [this coll k x])
  (before [this coll])
  (after [this coll]))

(defprotocol PrimaryIndex
  (get-by-key [this coll k])
  (get-all [this coll])
  (on-conflict [this old new]))

(defprotocol Compound
  :extend-via-metadata true
  (items [c])
  (add-items [c xs])
  (remove-keys [c ks]))

(defmulti indexer :index-type)

(defmethod indexer :one-to-one
  [opts]
  (let [{:keys [id kfn]} opts
        id (or id (when (keyword? kfn) kfn))]
    (assert (some? id) "Must provide an id")
    (reify
      PrimaryIndex
      (get-by-key [this coll k]
        (get coll k))
      (get-all [this coll]
        (vals coll))
      (on-conflict [this old new]
        new)
      Index
      (id [this]
        id)
      (extract-key [this x]
        (kfn x))
      (index [this coll k x]
        (assoc! coll k x))
      (unindex [this coll k x]
        (dissoc! coll k))
      (after [this coll]
        (persistent! coll))
      (before [this coll]
        (transient (or coll {}))))))

(defmethod indexer :one-to-many
  [opts]
  (let [{:keys [id kfn]} opts
        id (or id (when (keyword? kfn) kfn))]
    (assert (some? id) "Must provide an id")
    (reify
      Index
      (id [this]
        id)
      (extract-key [this x]
        (kfn x))
      (index [this coll k x]
        (let [existing (get coll k)]
          (assoc! coll k (conj (or existing #{}) x))))
      (unindex [this coll k x]
        (let [existing (get coll k)
              new (disj existing x)]
          (if (seq new)
            (assoc! coll k new)
            (dissoc! coll k))))
      (after [this coll]
        (persistent! coll))
      (before [this coll]
        (transient (or coll {}))))))

(defn dissoc-in
  [m [k & ks]]
  (if ks
    (let [m2 (dissoc-in (m k) ks)]
      (if (empty? m2)
        (dissoc m k)
        (assoc m k m2)))
    (dissoc m k)))

(defmethod indexer :nested-to-one
  [opts]
  (let [{:keys [id path]} opts
        id (or id (when (vector? path) path))]
    (assert (some? id) "Must provide an id")
    (reify
      Index
      (id [this]
        id)
      (extract-key [this x]
        (into [] (for [p path]
                   (p x))))
      (index [this coll path x]
        (assoc-in coll path x))
      (unindex [this coll path x]
        (dissoc-in coll path))
      (after [this coll]
        coll)
      (before [this coll]
        (or coll {})))))

(defmethod indexer :nested-to-many
  [opts]
  (let [{:keys [id path]} opts
        id (or id (when (vector? path) path))]
    (assert (some? id) "Must provide an id")
    (reify
      Index
      (id [this]
        id)
      (extract-key [this x]
        (into [] (for [p path]
                   (p x))))
      (index [this coll path x]
        (update-in coll path (fnil conj #{})))
      (unindex [this coll path x]
        (let [existing (get-in coll path)
              new (disj existing x)]
          (if (seq new)
            (assoc-in coll path new)
            (update-in coll (butlast path) dissoc (last path)))))
      (after [this coll]
        coll)
      (before [this coll]
        (or coll {})))))

(defmethod indexer :many-to-many
  [opts]
  (let [{:keys [id kfn]} opts
        id (or id (when (keyword? kfn) kfn))]
    (assert (some? id) "Must provide an id")
    (reify
      Index
      (id [this]
        id)
      (extract-key [this x]
        (kfn x))
      (index [this coll ks x]
        (reduce
         (fn [acc k]
           (assoc! acc k x))
         coll
         ks))
      (unindex [this coll ks x]
        (reduce
         (fn [acc k]
           (dissoc! acc k x))
         coll
         ks))
      (after [this coll]
        (persistent! coll))
      (before [this coll]
        (transient (or coll {}))))))

#?(:clj
   (defmacro compound [indexes]
     (let [[p-opts & s-opts] indexes
           pt-sym (gensym "pt")
           pi-sym (gensym "pi")
           st-syms (for [_ s-opts]
                     (gensym "st"))
           si-syms (for [_ s-opts]
                     (gensym "si"))
           m-sym (gensym "m")
           x-sym (gensym "x")
           k-sym (gensym "k")
           ex-sym (gensym "ex")]
       `(let [~pt-sym (indexer ~p-opts)
              ~@(mapcat (fn [sym opts]
                          [sym `(indexer ~opts)])
                        st-syms
                        s-opts)]
          (assert (satisfies? PrimaryIndex ~pt-sym) "First index must be a primary index")
          (with-meta {}
            {`items (fn [~m-sym]
                      (get-all ~pt-sym (get ~m-sym (id ~pt-sym))))
             `add-items (fn [~m-sym xs#]
                          (loop [~pi-sym (before ~pt-sym (get ~m-sym (id ~pt-sym)))
                                 ~@(mapcat (fn [st si]
                                             [si `(before ~st (get ~m-sym (id ~st)))])
                                           st-syms
                                           si-syms)
                                 [~x-sym & xs#] xs#]
                            (if (nil? ~x-sym)
                              (with-meta ~(into {`(id ~pt-sym) `(after ~pt-sym ~pi-sym)}
                                                (map (fn [st si]
                                                       [`(id ~st)
                                                        `(after ~st ~si)])
                                                     st-syms
                                                     si-syms))
                                (meta ~m-sym))
                              (let [k# (extract-key ~pt-sym ~x-sym)
                                    ~ex-sym (get-by-key ~pt-sym ~pi-sym k#)]
                                (if ~ex-sym
                                  (recur
                                   (index ~pt-sym ~pi-sym k# ~x-sym)
                                   ~@(map (fn [st si]
                                            `(let [k1# (extract-key ~st ~ex-sym)
                                                   k2# (extract-key ~st ~x-sym)]
                                               (index ~st (unindex ~st ~si k1# ~ex-sym) k2# ~x-sym)))
                                          st-syms
                                          si-syms)
                                   xs#)
                                  (recur
                                   (index ~pt-sym ~pi-sym k# ~x-sym)
                                   ~@(map (fn [st si]
                                            `(let [k# (extract-key ~st ~x-sym)]
                                               (index ~st ~si k# ~x-sym)))
                                          st-syms
                                          si-syms)
                                   xs#))))))
             `remove-keys (fn [~m-sym ks#]
                            (loop [~pi-sym (before ~pt-sym (get ~m-sym (id ~pt-sym)))
                                   ~@(mapcat (fn [st si]
                                               [si `(before ~st (get ~m-sym (id ~st)))])
                                             st-syms
                                             si-syms)
                                   [~k-sym & ks#] ks#]
                              (if (nil? ~k-sym)
                                (with-meta ~(into {`(id ~pt-sym) `(after ~pt-sym ~pi-sym)}
                                                  (map (fn [st si]
                                                         [`(id ~st)
                                                          `(after ~st ~si)])
                                                       st-syms
                                                       si-syms))
                                  (meta ~m-sym))
                                (if-let [~ex-sym (get-by-key ~pt-sym ~pi-sym ~k-sym)]
                                  (recur
                                   (unindex ~pt-sym ~pi-sym ~k-sym ~ex-sym)
                                   ~@(map (fn [st si]
                                            `(let [k# (extract-key ~st ~ex-sym)]
                                               (unindex ~st ~si k# ~ex-sym)))
                                          st-syms
                                          si-syms)
                                   ks#)
                                  (recur
                                   ~pi-sym
                                   ~@si-syms
                                   ks#)))))})))))

(defn compound* [indexes]
  (let [[pi & sis] (doall (map indexer indexes))]
    (assert (satisfies? PrimaryIndex pi) "First index must be a primary index")
    (with-meta {}
      {`items (fn [c]
                (get-all pi (get c (id pi))))
       `add-items (fn [c xs]
                    (loop [px (before pi (get c (id pi)))
                           sixs (reduce (fn [acc si]
                                          (assoc acc si (before si (get c (id si)))))
                                        {}
                                        sis)
                           [x & xs] xs]
                      (if (nil? x)
                        (with-meta
                          (reduce-kv (fn [acc si sx]
                                       (assoc acc (id si) (after si sx)))
                                     {(id pi) (after pi px)}
                                     sixs)
                          (meta c))
                        (let [k (extract-key pi x)
                              ex (get-by-key pi px k)]
                          (if ex
                            (recur (index pi px k (on-conflict pi ex x))
                                   (reduce-kv (fn [acc si sx]
                                                (assoc acc si (let [k1 (extract-key si ex)
                                                                    k2 (extract-key si x)]
                                                                (index si (unindex si sx k1 ex) k2 x))))
                                              {}
                                              sixs)
                                   xs)
                            (recur (index pi px k x)
                                   (reduce-kv (fn [acc si sx]
                                                (assoc acc si (let [k (extract-key si x)]
                                                                (index si sx k x))))
                                              {}
                                              sixs)
                                   xs))))))
       `remove-keys (fn [c ks]
                      (loop [px (before pi (get c (id pi)))
                             sixs (reduce (fn [acc si]
                                            (assoc acc si (before si (get c (id si)))))
                                          {}
                                          sis)
                             [k & ks] ks]
                        (if (nil? k)
                          (with-meta
                            (reduce-kv (fn [acc si sx]
                                         (assoc acc (id si) (after si sx)))
                                       {(id pi) (after pi px)}
                                       sixs)
                            (meta c))
                          (if-let [ex (get-by-key pi px k)]
                            (recur (unindex pi px k ex)
                                   (reduce-kv (fn [acc si sx]
                                                (assoc acc si (let [k (extract-key si ex)]
                                                                (unindex si sx k ex))))
                                              {}
                                              sixs)
                                   ks)
                            (recur px sixs ks)))))})))
{'fraction_non_alphanumeric': 0.11613281905081335, 'fraction_numerical': 0.001006204930404159, 'mean_word_length': 0.7421121822962313, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 269, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3065328', 'n_tokens_mistral': 6364, 'n_tokens_neox': 6212, 'n_words': 1980}