<commit_before>;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;                                                                  ;;;
;;; Free Software published under an MIT-like license. See LICENSE   ;;;
;;;                                                                  ;;;
;;; Copyright (c) 2012 Google, Inc.  All rights reserved.            ;;;
;;;                                                                  ;;;
;;; Original author: <NAME>                                ;;;
;;;                                                                  ;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(in-package :mysqlnd)

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; Packet Definition Macrology

#||
Specs:

slot-specifier ::= (slot-name [[slot-options]])
slot-name ::= symbol
slot-options ::= {:bind boolean } |
                 {:eof eof-action} |
                 {:mysql-type parser-type-spec} |
                 {:predicate form} |
                 {:reduce λ α′ α → β)
                 {:transform λ α → β} |
                 {:transient boolean} |
                 {:type type-specifier} |
                 {:value form}

parser-type-spec ::= (integer integer-size) |
                     (string-type string-termination-spec)

integer-size ::= byte-count | :lenenc
 byte-count - read this many bytes
 :lenenc - read a length-encoded integer

string-type ::= octets | string
 octets - '(vector (unsigned-byte 8))
 string - shorthand for octets transformed with #'babel:octets-to-string.
          NB: this is separate from the :transform option.

string-termination-spec ::= integer | :eof | :lenenc | :null | :null-eof
 integer - a specific length.
 :eof - read until the end of the packet.
 :lenenc - read a length-encoded integer first, then use that as the length.
 :null - read until a null byte is encountered.
 :null-eof - read until a null byte is encountered or we hit the end of the packet.
             Used to deal with a bug in some forms of the initial handshake packet.

eof-action ::= :error | :end
 :error - default; end-of-file signaled
 :end - stop parsing packet and return collected data

Bind - Bind the slot value to its name so later slots may use it.
Predicate - The provided form must return non-nil for this slot to be parsed as described.
Reduce - The parsed value is combined with a previously parsed value of the same name using the λ provided.
         The old value is passed as the first argument; the new value is passed as the second argument.
Transform - The parsed value is transformed using the provided λ.
Transient - The parsed value is not returned as a parsed value. It may be used internally if named.
Value - If present, the value parsed for this slot is expected to be equal to the value of the form.

The Lisp type specified by :TYPE may be omitted if it can be deduced from :MYSQL-TYPE. Generally, if
:TRANSFORM or :REDUCE are specified, you should specify a :TYPE.

Order of Operations:

• Predicate
• parse-from-stream
• Transform
• Value
• Reduce
• Bind
||#

#|
TODO:
1) bind by default (rename bind?)
2) bind first, make struct later
3) add lisp-type override to packet slots; require if transform or reduce is present
4) make struct types stricter
|#

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Slot meta-data
(defclass packet-slot ()
  ((name :type symbol
         :initarg :name
         :reader packet-slot-name)
   (bind :type boolean
         :initarg :bind
         :initform t
         :accessor packet-slot-bind)
   (eof  :type keyword
         :initarg :eof
         :initform :error
         :accessor packet-slot-eof)
   (predicate :type (or function null)
              :initform nil
              :initarg :predicate
              :accessor packet-slot-predicate)
   (reduce :type (or function null)
              :initform nil
           :initarg :reduce
           :accessor packet-slot-reduce)
   (transform :type (or function null)
              :initform nil
              :initarg :transform
              :accessor packet-slot-transform)
   (transient :type boolean
              :initarg :transient
              :initform nil
              :accessor packet-slot-transient)
   (mysql-type :initarg :mysql-type
               :accessor packet-slot-mysql-type)
   (type :initarg :type
         :initform nil
         :accessor packet-slot-type)
   (value :initarg :value
          :initform nil
          :accessor packet-slot-value)))

(defun parse-slot (slot-specifier)
  (destructuring-bind (slot-name &rest slot-properties)
      slot-specifier
    (apply #'make-instance
           'packet-slot
           :name slot-name
           slot-properties)))
<commit_msg>define-packet: drop type-specification for predicate, reduce, and transform<commit_after>;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;                                                                  ;;;
;;; Free Software published under an MIT-like license. See LICENSE   ;;;
;;;                                                                  ;;;
;;; Copyright (c) 2012 Google, Inc.  All rights reserved.            ;;;
;;;                                                                  ;;;
;;; Original author: <NAME>                                ;;;
;;;                                                                  ;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(in-package :mysqlnd)

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;; Packet Definition Macrology

#||
Specs:

slot-specifier ::= (slot-name [[slot-options]])
slot-name ::= symbol
slot-options ::= {:bind boolean } |
                 {:eof eof-action} |
                 {:mysql-type parser-type-spec} |
                 {:predicate form} |
                 {:reduce λ α′ α → β)
                 {:transform λ α → β} |
                 {:transient boolean} |
                 {:type type-specifier} |
                 {:value form}

parser-type-spec ::= (integer integer-size) |
                     (string-type string-termination-spec)

integer-size ::= byte-count | :lenenc
 byte-count - read this many bytes
 :lenenc - read a length-encoded integer

string-type ::= octets | string
 octets - '(vector (unsigned-byte 8))
 string - shorthand for octets transformed with #'babel:octets-to-string.
          NB: this is separate from the :transform option.

string-termination-spec ::= integer | :eof | :lenenc | :null | :null-eof
 integer - a specific length.
 :eof - read until the end of the packet.
 :lenenc - read a length-encoded integer first, then use that as the length.
 :null - read until a null byte is encountered.
 :null-eof - read until a null byte is encountered or we hit the end of the packet.
             Used to deal with a bug in some forms of the initial handshake packet.

eof-action ::= :error | :end
 :error - default; end-of-file signaled
 :end - stop parsing packet and return collected data

Bind - Bind the slot value to its name so later slots may use it.
Predicate - The provided form must return non-nil for this slot to be parsed as described.
Reduce - The parsed value is combined with a previously parsed value of the same name using the λ provided.
         The old value is passed as the first argument; the new value is passed as the second argument.
Transform - The parsed value is transformed using the provided λ.
Transient - The parsed value is not returned as a parsed value. It may be used internally if named.
Value - If present, the value parsed for this slot is expected to be equal to the value of the form.

The Lisp type specified by :TYPE may be omitted if it can be deduced from :MYSQL-TYPE. Generally, if
:TRANSFORM or :REDUCE are specified, you should specify a :TYPE.

Order of Operations:

• Predicate
• parse-from-stream
• Transform
• Value
• Reduce
• Bind
||#

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Slot meta-data
(defclass packet-slot ()
  ((name :type symbol
         :initarg :name
         :reader packet-slot-name)
   (bind :type boolean
         :initarg :bind
         :initform t
         :accessor packet-slot-bind)
   (eof  :type keyword
         :initarg :eof
         :initform :error
         :accessor packet-slot-eof)
   (predicate :initform nil
              :initarg :predicate
              :accessor packet-slot-predicate)
   (reduce :initform nil
           :initarg :reduce
           :accessor packet-slot-reduce)
   (transform :initform nil
              :initarg :transform
              :accessor packet-slot-transform)
   (transient :type boolean
              :initarg :transient
              :initform nil
              :accessor packet-slot-transient)
   (mysql-type :initarg :mysql-type
               :accessor packet-slot-mysql-type)
   (type :initarg :type
         :initform nil
         :accessor packet-slot-type)
   (value :initarg :value
          :initform nil
          :accessor packet-slot-value)))

(defun parse-slot (slot-specifier)
  (destructuring-bind (slot-name &rest slot-properties)
      slot-specifier
    (apply #'make-instance
           'packet-slot
           :name slot-name
           slot-properties)))

{'fraction_non_alphanumeric': 0.15009642168416543, 'fraction_numerical': 0.0014998928647953717, 'mean_word_length': 2.1241633199464527, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3172653', 'n_tokens_mistral': 2665, 'n_tokens_neox': 2453, 'n_words': 911}