<commit_before>(in-package :bert)


(defgeneric translate-complex-type (object)
  (:documentation "Translates tuples with the 'bert' tag to corresponding Lisp objects."))

(defgeneric encode (object &key berp-header)
  (:documentation "Encodes the BERT-translatable object to a vector of bytes."))


(define-constant true (tuple '|bert| '|true|)
  :test #'match-p
  :documentation "BERT boolean true term.")

(define-constant false (tuple '|bert| '|false|)
  :test #'match-p
  :documentation "BERT boolean false term.")

(defun bool (value)
  (if value true false))


(defclass bert-time ()
  ((megaseconds :reader megaseconds :initarg :megaseconds)
   (seconds :reader seconds :initarg :seconds)
   (microseconds :reader microseconds :initarg :microseconds))
  (:documentation "BERT time data type"))

(defmethod translate-complex-type ((time bert-time))
  (with-slots (megaseconds seconds microseconds) time
    (tuple '|bert| '|time| megaseconds seconds microseconds)))


(defclass bert-regex ()
  ((source :reader regex-source :initarg :source)
   (options :reader regex-options :initarg :options))
  (:documentation "BERT regex data type"))

(defmethod translate-complex-type ((regex bert-regex))
  (with-slots (source options) regex
    (tuple '|bert| '|regex| (string-to-binary source) options)))


(defmethod translate-complex-type ((dict hash-table))
  (tuple '|bert| '|dict|
         (loop
            for key being the hash-keys in dict using (hash-value value)
            collect (tuple (translate-complex-type key)
                           (translate-complex-type value))) ))


(defmethod translate-complex-type ((nil-symbol (eql nil)))
  (declare (ignorable nil-symbol))
  (tuple '|bert| '|nil|))


(defmethod translate-complex-type ((tuple erlang-tuple))
  (with-slots (elements) tuple
    (make-instance 'erlang-tuple
                   :elements (map 'vector #'translate-complex-type elements))))


(defmethod translate-complex-type ((lst list))
  (mapcar #'translate-complex-type lst))


(defmethod translate-complex-type (object)
  object)


(defmethod encode (object &key berp-header)
  (let ((bytes (erlang-term:encode (translate-complex-type object)
                              :version-tag +protocol-version+)))
    (if berp-header
        (concatenate '(vector (unsigned-byte 8))
                     (uint32-to-bytes (length bytes))
                     bytes)
        bytes)))

<commit_msg>Tidied up a little.<commit_after>(in-package :bert)


(defgeneric translate-complex-type (object)
  (:documentation "Translates tuples with the 'bert' tag to corresponding Lisp objects."))

(defgeneric encode (object &key berp-header)
  (:documentation "Encodes the BERT-translatable object to a vector of bytes."))


(define-constant true (tuple '|bert| '|true|)
  :test #'match-p
  :documentation "BERT boolean true term.")

(define-constant false (tuple '|bert| '|false|)
  :test #'match-p
  :documentation "BERT boolean false term.")

(defun bool (value)
  (if value true false))


(defclass bert-time ()
  ((megaseconds :reader megaseconds :initarg :megaseconds)
   (seconds :reader seconds :initarg :seconds)
   (microseconds :reader microseconds :initarg :microseconds))
  (:documentation "BERT time data type"))

(defmethod translate-complex-type ((time bert-time))
  (with-slots (megaseconds seconds microseconds) time
    (tuple '|bert| '|time| megaseconds seconds microseconds)))


(defclass bert-regex ()
  ((source :reader regex-source :initarg :source)
   (options :reader regex-options :initarg :options))
  (:documentation "BERT regex data type"))

(defmethod translate-complex-type ((regex bert-regex))
  (with-slots (source options) regex
    (tuple '|bert| '|regex| (string-to-binary source) options)))


(defmethod translate-complex-type ((dict hash-table))
  (tuple '|bert| '|dict|
         (loop
            for key being the hash-keys in dict using (hash-value value)
            collect (tuple (translate-complex-type key)
                           (translate-complex-type value))) ))


(defmethod translate-complex-type ((nil-symbol (eql nil)))
  (declare (ignorable nil-symbol))
  (tuple '|bert| '|nil|))


(defmethod translate-complex-type ((tuple erlang-tuple))
  (with-slots (elements) tuple
    (make-instance 'erlang-tuple
                   :elements (map 'vector #'translate-complex-type elements))))


(defmethod translate-complex-type ((lst list))
  (mapcar #'translate-complex-type lst))


(defmethod translate-complex-type (object)
  object)


(defmethod encode (object &key berp-header)
  (let ((bytes (erlang-term:encode (translate-complex-type object)
                                   :version-tag +protocol-version+)))
    (when berp-header
      (concatenate '(vector (unsigned-byte 8))
                   (uint32-to-bytes (length bytes))
                   bytes))
    bytes))


{'fraction_non_alphanumeric': 0.13353973168214656, 'fraction_numerical': 0.001238390092879257, 'mean_word_length': 3.503717472118959, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1247693', 'n_tokens_mistral': 1515, 'n_tokens_neox': 1450, 'n_words': 429}