<commit_before>(ns morpheus.models.edge.base
  (:require [morpheus.utils :refer :all]
            [morpheus.models.base :as mb]
            [morpheus.models.core :as mc]
            [neb.core :as neb]
            [neb.cell :as neb-cell]
            [neb.header :as neb-header]
            [cluster-connector.utils.for-debug :refer [spy $]]
            [cluster-connector.native-cache.core :refer [defcache evict-cache-key]]
            [neb.utils :refer [map-on-vals]])
  (:import (java.util UUID)
           (org.shisoft.neb Trunk)
           (org.shisoft.neb.io type_lengths CellMeta Reader)
           (com.google.common.hash Hashing)))

(def schema-fields
  [[:*start*  :cid]
   [:*end*  :cid]])

(defmulties
  :type
  (edge-base-schema [])
  (v1-vertex-field [])
  (v2-vertex-field [])
  (type-stick-body [])
  (vertex-fields [])
  (delete-edge-cell [edge start end]))

(defmulties
  :body
  (update-edge [id func-sym params])
  (delete-edge [])
  (base-schema [])
  (require-edge-cell? [])
  (edge-schema [base-schema fields])
  (create-edge-cell [vertex-fields & args])
  (edges-from-cid-array [cid-list & [start-vertex]]))

(defn edge-cell-vertex-fields [v1 v2]
  {:*start* v1
   :*end* v2})

(def vertex-fields #{:*start* :*end*})

(def max-list-size (-> (Trunk/getMaxObjSize)
                       (- neb-header/cell-head-len)
                       (/ type_lengths/cidLen)
                       (Math/floor) (int)
                       (dec)))

(defn extract-edge-cid-list [cid-lists edge-schema-id]
  (first (filter
           (fn [item]
             (= edge-schema-id (:sid item)))
           cid-lists)))

(defn extract-cell-list-id [vertex field edge-schema-id]
  (let [cid-lists (get vertex field)]
    (:list-cid (extract-edge-cid-list cid-lists edge-schema-id))))

(defn edge-list* [[v-id field edge-schema-id]]
  (UUID.
    (.getMostSignificantBits v-id)
    (first (neb/hash-str (str v-id "-" field "-" edge-schema-id)))))

(defcache edge-list {:expire-after-access-secs 60
                     :soft-values? true} edge-list*)

(defn cid-list-id-by-vertex [v-id field edge-schema-id]
  (edge-list [v-id field edge-schema-id]))

(def empty-cid (UUID. 0 0))

(defn format-edge-cells [group-props direction edge]
  (let [pure-edge (dissoc edge :*schema* :*hash*)]
    (merge pure-edge
           {:*ep* group-props
            :*direction* direction})))

(defmacro with-cid-list [& body]
  `(let [^Trunk ~'trunk neb-cell/*cell-trunk*
         ^Integer ~'hash neb-cell/*cell-hash*
         ^CellMeta ~'meta neb-cell/*cell-meta*
         ~'next-cid (neb-cell/get-in-cell ~'trunk ~'hash [:next-list])
         ~'next-cid (when-not (= ~'next-cid empty-cid) ~'next-cid)]
     ~@body))

(defn read-cid-list-len []
  (Reader/readInt
    (+ (.getLocation neb-cell/*cell-meta*)
       type_lengths/cidLen
       neb-header/cell-head-len)))

(defmacro def-cid-append-op [func-name params & body]
  `(defn ~func-name ~params
     (with-cid-list
       (let [~'move-to-list-with-params (fn [cid-id# params#]
                                          (apply neb/write-lock-exec* cid-id#
                                                 (quote ~(symbol (str "morpheus.models.edge.base/"
                                                                      (name func-name))))
                                                 params#))
             ~'move-to-list (fn [next-cid#] (~'move-to-list-with-params next-cid# ~params))
             ~'list-length (read-cid-list-len)]
         ~@body))))

(defn new-list-cell []
  (let [cell-id (neb/new-cell-by-ids (neb/rand-cell-id) @mb/cid-list-schema-id
                                     {:next-list empty-cid :cid-array []})]
    (neb-cell/update-cell*
      neb-cell/*cell-trunk* neb-cell/*cell-hash*
      (fn [cid-list]
        (assoc cid-list :next-list cell-id)))
    cell-id))

(def-cid-append-op<commit_msg>use long long map for neb<commit_after>(ns morpheus.models.edge.base
  (:require [morpheus.utils :refer :all]
            [morpheus.models.base :as mb]
            [morpheus.models.core :as mc]
            [neb.core :as neb]
            [neb.cell :as neb-cell]
            [neb.header :as neb-header]
            [cluster-connector.utils.for-debug :refer [spy $]]
            [cluster-connector.native-cache.core :refer [defcache evict-cache-key]]
            [neb.utils :refer [map-on-vals]])
  (:import (java.util UUID)
           (org.shisoft.neb Trunk)
           (org.shisoft.neb.io type_lengths Reader)
           (com.google.common.hash Hashing)))

(def schema-fields
  [[:*start*  :cid]
   [:*end*  :cid]])

(defmulties
  :type
  (edge-base-schema [])
  (v1-vertex-field [])
  (v2-vertex-field [])
  (type-stick-body [])
  (vertex-fields [])
  (delete-edge-cell [edge start end]))

(defmulties
  :body
  (update-edge [id func-sym params])
  (delete-edge [])
  (base-schema [])
  (require-edge-cell? [])
  (edge-schema [base-schema fields])
  (create-edge-cell [vertex-fields & args])
  (edges-from-cid-array [cid-list & [start-vertex]]))

(defn edge-cell-vertex-fields [v1 v2]
  {:*start* v1
   :*end* v2})

(def vertex-fields #{:*start* :*end*})

(def max-list-size (-> (Trunk/getMaxObjSize)
                       (- neb-header/cell-head-len)
                       (/ type_lengths/cidLen)
                       (Math/floor) (int)
                       (dec)))

(defn extract-edge-cid-list [cid-lists edge-schema-id]
  (first (filter
           (fn [item]
             (= edge-schema-id (:sid item)))
           cid-lists)))

(defn extract-cell-list-id [vertex field edge-schema-id]
  (let [cid-lists (get vertex field)]
    (:list-cid (extract-edge-cid-list cid-lists edge-schema-id))))

(defn edge-list* [[v-id field edge-schema-id]]
  (UUID.
    (.getMostSignificantBits v-id)
    (first (neb/hash-str (str v-id "-" field "-" edge-schema-id)))))

(defcache edge-list {:expire-after-access-secs 60
                     :soft-values? true} edge-list*)

(defn cid-list-id-by-vertex [v-id field edge-schema-id]
  (edge-list [v-id field edge-schema-id]))

(def empty-cid (UUID. 0 0))

(defn format-edge-cells [group-props direction edge]
  (let [pure-edge (dissoc edge :*schema* :*hash*)]
    (merge pure-edge
           {:*ep* group-props
            :*direction* direction})))

(defmacro with-cid-list [& body]
  `(let [^Trunk ~'trunk neb-cell/*cell-trunk*
         ^Integer ~'hash neb-cell/*cell-hash*
         ~'cell-addr neb-cell/*cell-addr*
         ~'next-cid (neb-cell/get-in-cell ~'trunk ~'hash [:next-list])
         ~'next-cid (when-not (= ~'next-cid empty-cid) ~'next-cid)]
     ~@body))

(defn read-cid-list-len []
  (Reader/readInt
    (+ neb-cell/*cell-addr*
       type_lengths/cidLen
       neb-header/cell-head-len)))

(defmacro def-cid-append-op [func-name params & body]
  `(defn ~func-name ~params
     (with-cid-list
       (let [~'move-to-list-with-params (fn [cid-id# params#]
                                          (apply neb/write-lock-exec* cid-id#
                                                 (quote ~(symbol (str "morpheus.models.edge.base/"
                                                                      (name func-name))))
                                                 params#))
             ~'move-to-list (fn [next-cid#] (~'move-to-list-with-params next-cid# ~params))
             ~'list-length (read-cid-list-len)]
         ~@body))))

(defn new-list-cell []
  (let [cell-id (neb/new-cell-by-ids (neb/rand-cell-id) @mb/cid-list-schema-id
                                     {:next-list empty-cid :cid-array []})]
    (neb-cell/update-cell*
      neb-cell/*cell-trunk* neb-cell/*cell-hash*
      (fn [cid-list]
        (assoc cid-list :next-list cell-id)))
    cell-id))

(def-cid-append-op
{'fraction_non_alphanumeric': 0.17316409791477788, 'fraction_numerical': 0.0025903380391141045, 'mean_word_length': 1.9574875526618154, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6872269', 'n_tokens_mistral': 2790, 'n_tokens_neox': 2749, 'n_words': 521}