<commit_before>(in-package :btcl-net)

(bindata:define-binary-type raw-bytes (size)
  (:reader (in)
           (let ((buf (make-array size :element-type '(unsigned-byte 8))))
             (read-sequence buf in)
             buf))
  (:writer (out buf)
           (write-sequence buf out)))

(bindata:define-binary-type unsigned-integer-be (bytes)
  (:reader (in)
           (loop with value = 0
              for low-bit downfrom (* 8 (1- bytes)) to 0 by 8 do
                (setf (ldb (byte 8 low-bit) value) (read-byte in))
              finally (return value)))
  (:writer (out value)
           (loop for low-bit downfrom (* 8 (1- bytes)) to 0 by 8
              do (write-byte (ldb (byte 8 low-bit) value) out))))
(bindata:define-binary-type u16be () (unsigned-integer-be :bytes 2))
(bindata:define-binary-type u32be () (unsigned-integer-be :bytes 4))
(bindata:define-binary-type u64be () (unsigned-integer-be :bytes 8))

(defun read-unsigned-le (in bytes)
  (loop with value = 0
     for low-bit upto (* 8 (1- bytes)) from 0 by 8 do
       (setf (ldb (byte 8 low-bit) value) (read-byte in))
     finally (return value)))
(defun write-unsigned-le (out value bytes)
  (loop for low-bit upto (* 8 (1- bytes)) from 0 by 8
     do (write-byte (ldb (byte 8 low-bit) value) out)))

(bindata:define-binary-type unsigned-integer-le (bytes)
  (:reader (in) (read-unsigned-le in bytes))
  (:writer (out value) (write-unsigned-le out value bytes)))
(bindata:define-binary-type u8 () (unsigned-integer-le :bytes 1))
(bindata:define-binary-type u16le () (unsigned-integer-le :bytes 2))
(bindata:define-binary-type u32le () (unsigned-integer-le :bytes 4))
(bindata:define-binary-type u64le () (unsigned-integer-le :bytes 8))

(bindata:define-binary-type signed-integer-le (bytes)
  (:reader (in)
           (let ((rawnum (read-unsigned-le in bytes))
                 (bytes-max (expt 2 bytes)))
             (if (>= rawnum (/ bytes-max 2))
                 (- bytes-max rawnum)
                 rawnum)))
  (:writer (out value)
           (if (>= value 0)
               (write-unsigned-le out value bytes)
               (error "Writing negative values isn't implemented yet lulz"))))

(bindata:define-binary-type s32le () (signed-integer-le :bytes 4))
(bindata:define-binary-type s64le () (signed-integer-le :bytes 8))

;; this is CompactSize in BitcoinQT
(bindata:define-binary-type varint ()
  (:reader (in)
           (let ((lowbyte (read-byte in)))
             (cond ((< lowbyte #xfd) lowbyte)
                   ((= lowbyte #xfd) (read-unsigned-le in 2))
                   ((= lowbyte #xfe) (read-unsigned-le in 4))
                   ((= lowbyte #xff) (read-unsigned-le in 8)))))
  (:writer (out value)
           (cond ((< value #xfd) (write-unsigned-le out value 1))
                 ((<= value #xffff) (progn (write-byte #xfd out)
                                           (write-unsigned-le out value 2)))
                 ((<= value #xffffffff) (progn (write-byte #xfe out)
                                               (write-unsigned-le out value 4)))
                 (t (progn (write-byte #xff out)
                           (write-unsigned-le out value 8))))))

(bindata:define-binary-type generic-string (length character-type)
  (:reader (in)
           (let ((string (make-string length))
                 (end 0))
             (loop for i from 0 below length
                for char = (bindata:read-value character-type in)
                do (setf (char string i) char)
                unless (char= char #\null) do (incf end) end
                finally (return (subseq string 0 end)))))
  (:writer (out string)
           (dotimes (i (length string))
             (bindata:write-value character-type out (char string i)))
           (if (> length (length string))
               (dotimes (i (- length (length string)))
                 (write-byte 0 out)))))

(bindata:define-binary-type iso-8859-1-char ()
  (:reader (in)
           (let ((code (read-byte in)))
             (or (code-char code)
                 (error "Character code ~d not supported" code))))
  (:writer (out char)
           (let ((code (char-code char)))
             (if (<= 0 code #xff)
                 (write-byte code out)
                 (error "Illegal character for iso-8859-1 encoding: character: ~c with code: ~d" char code)))))

(bindata:define-binary-type iso-8859-1-string (length)
  (generic-string :length length :character-type 'iso-8859-1-char))

(bindata:define-binary-class varstr ()
  ((len varint)
   (str (iso-8859-1-string :length len))))

(bindata:define-binary-class version-net-addr ()
  ((services u64le)
   (ip-addr (raw-bytes :size 16))
   (port u16be)))

(bindata:define-binary-class net-addr ()
  ((timestamp u32le)
   (services u64le)
   (ip-addr (raw-bytes :size 16))
   (port u16be)))

(bindata:define-binary-type addr-list (count)
  (:reader (in)
           (loop for i from 0 upto count
              for addr = (bindata:read-value 'net-addr in)
              while addr
              collect addr))
  (:writer (out net-addrs)
           (dolist (net-addr net-addrs)
             (bindata:write-value 'net-addr out net-addr))))

(bindata:define-binary-class inv-vector ()
  ((obj-type u32le)
   (hash (raw-bytes :size 32))))

(bindata:define-binary-type inv-vector-list (count)
  (:reader (in)
           (loop for i from 0 upto count
              for inv-vector = (bindata:read-value 'inv-vector in)
              while inv-vector
              collect inv-vector))
  (:writer (out inv-vectors)
           (dolist (inv-vec inv-vectors)
             (bindata:write-value 'inv-vector out inv-vec))))

(defun make-varstr (str)
  (make-instance 'varstr
                 :len (length str)
                 :str str))

(defgeneric build-ip-addr (addr)
  (:documentation "make a 16-byte address in network byte order"))

;; (defmethod build-ip-addr ((addr string))
;;   (let ((result 0))
;;     (setf (ldb (byte 8 80) result) 255) ;;;; 0xFFFF to represent IPv4
;;     (setf (ldb (byte 8 88) result) 255) ;;;; address within IPv6 address<commit_msg>fix reading lists of things<commit_after>(in-package :btcl-net)

(bindata:define-binary-type raw-bytes (size)
  (:reader (in)
           (let ((buf (make-array size :element-type '(unsigned-byte 8))))
             (read-sequence buf in)
             buf))
  (:writer (out buf)
           (write-sequence buf out)))

(bindata:define-binary-type unsigned-integer-be (bytes)
  (:reader (in)
           (loop with value = 0
              for low-bit downfrom (* 8 (1- bytes)) to 0 by 8 do
                (setf (ldb (byte 8 low-bit) value) (read-byte in))
              finally (return value)))
  (:writer (out value)
           (loop for low-bit downfrom (* 8 (1- bytes)) to 0 by 8
              do (write-byte (ldb (byte 8 low-bit) value) out))))
(bindata:define-binary-type u16be () (unsigned-integer-be :bytes 2))
(bindata:define-binary-type u32be () (unsigned-integer-be :bytes 4))
(bindata:define-binary-type u64be () (unsigned-integer-be :bytes 8))

(defun read-unsigned-le (in bytes)
  (loop with value = 0
     for low-bit upto (* 8 (1- bytes)) from 0 by 8 do
       (setf (ldb (byte 8 low-bit) value) (read-byte in))
     finally (return value)))
(defun write-unsigned-le (out value bytes)
  (loop for low-bit upto (* 8 (1- bytes)) from 0 by 8
     do (write-byte (ldb (byte 8 low-bit) value) out)))

(bindata:define-binary-type unsigned-integer-le (bytes)
  (:reader (in) (read-unsigned-le in bytes))
  (:writer (out value) (write-unsigned-le out value bytes)))
(bindata:define-binary-type u8 () (unsigned-integer-le :bytes 1))
(bindata:define-binary-type u16le () (unsigned-integer-le :bytes 2))
(bindata:define-binary-type u32le () (unsigned-integer-le :bytes 4))
(bindata:define-binary-type u64le () (unsigned-integer-le :bytes 8))

(bindata:define-binary-type signed-integer-le (bytes)
  (:reader (in)
           (let ((rawnum (read-unsigned-le in bytes))
                 (bytes-max (expt 2 bytes)))
             (if (>= rawnum (/ bytes-max 2))
                 (- bytes-max rawnum)
                 rawnum)))
  (:writer (out value)
           (if (>= value 0)
               (write-unsigned-le out value bytes)
               (error "Writing negative values isn't implemented yet lulz"))))

(bindata:define-binary-type s32le () (signed-integer-le :bytes 4))
(bindata:define-binary-type s64le () (signed-integer-le :bytes 8))

;; this is CompactSize in BitcoinQT
(bindata:define-binary-type varint ()
  (:reader (in)
           (let ((lowbyte (read-byte in)))
             (cond ((< lowbyte #xfd) lowbyte)
                   ((= lowbyte #xfd) (read-unsigned-le in 2))
                   ((= lowbyte #xfe) (read-unsigned-le in 4))
                   ((= lowbyte #xff) (read-unsigned-le in 8)))))
  (:writer (out value)
           (cond ((< value #xfd) (write-unsigned-le out value 1))
                 ((<= value #xffff) (progn (write-byte #xfd out)
                                           (write-unsigned-le out value 2)))
                 ((<= value #xffffffff) (progn (write-byte #xfe out)
                                               (write-unsigned-le out value 4)))
                 (t (progn (write-byte #xff out)
                           (write-unsigned-le out value 8))))))

(bindata:define-binary-type generic-string (length character-type)
  (:reader (in)
           (let ((string (make-string length))
                 (end 0))
             (loop for i from 0 below length
                for char = (bindata:read-value character-type in)
                do (setf (char string i) char)
                unless (char= char #\null) do (incf end) end
                finally (return (subseq string 0 end)))))
  (:writer (out string)
           (dotimes (i (length string))
             (bindata:write-value character-type out (char string i)))
           (if (> length (length string))
               (dotimes (i (- length (length string)))
                 (write-byte 0 out)))))

(bindata:define-binary-type iso-8859-1-char ()
  (:reader (in)
           (let ((code (read-byte in)))
             (or (code-char code)
                 (error "Character code ~d not supported" code))))
  (:writer (out char)
           (let ((code (char-code char)))
             (if (<= 0 code #xff)
                 (write-byte code out)
                 (error "Illegal character for iso-8859-1 encoding: character: ~c with code: ~d" char code)))))

(bindata:define-binary-type iso-8859-1-string (length)
  (generic-string :length length :character-type 'iso-8859-1-char))

(bindata:define-binary-class varstr ()
  ((len varint)
   (str (iso-8859-1-string :length len))))

(bindata:define-binary-class version-net-addr ()
  ((services u64le)
   (ip-addr (raw-bytes :size 16))
   (port u16be)))

(bindata:define-binary-class net-addr ()
  ((timestamp u32le)
   (services u64le)
   (ip-addr (raw-bytes :size 16))
   (port u16be)))

(bindata:define-binary-type addr-list (count)
  (:reader (in)
           (loop repeat count
              collect (bindata:read-value 'net-addr in)))
  (:writer (out net-addrs)
           (dolist (net-addr net-addrs)
             (bindata:write-value 'net-addr out net-addr))))

(bindata:define-binary-class inv-vector ()
  ((obj-type u32le)
   (hash (raw-bytes :size 32))))

(bindata:define-binary-type inv-vector-list (count)
  (:reader (in)
           (loop repeat count
              collect (bindata:read-value 'inv-vector in)))
  (:writer (out inv-vectors)
           (dolist (inv-vec inv-vectors)
             (bindata:write-value 'inv-vector out inv-vec))))

(defun make-varstr (str)
  (make-instance 'varstr
                 :len (length str)
                 :str str))

(defgeneric build-ip-addr (addr)
  (:documentation "make a 16-byte address in network byte order"))

;; (defmethod build-ip-addr ((addr string))
;;   (let ((result 0))
;;     (setf (ldb (byte 8 80) result) 255) ;;;; 0xFFFF to represent IPv4
;;     (setf (ldb (byte 8 88) result) 255) ;;;; address within IPv6 address
{'fraction_non_alphanumeric': 0.14707840852338938, 'fraction_numerical': 0.02097552855002497, 'mean_word_length': 2.2160064239828694, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4762301', 'n_tokens_mistral': 4009, 'n_tokens_neox': 3746, 'n_words': 1133}