<commit_before>(in-package :mgl-mat)

;;;; Float I/O

(defun write-as-bytes (integer n stream)
  (let ((x integer))
    (loop repeat n do
      (write-byte (logand x #xff) stream)
      (setq x (ash x -8)))
    (assert (zerop x))))

(defun read-as-bytes (n stream)
  (let ((x 0))
    (loop for i below n do
      (incf x (ash (read-byte stream) (* i 8))))
    x))

(defun write-single-float-array (array stream &key (start 0)
                                 (end (array-total-size array)))
  (loop for i upfrom start below end do
    (write-as-bytes (ieee-floats:encode-float32 (row-major-aref array i))
                    4 stream)))

(defun write-double-float-array (array stream &key (start 0)
                                 (end (array-total-size array)))
  (loop for i upfrom start below end do
    (write-as-bytes (ieee-floats:encode-float64 (row-major-aref array i))
                    8 stream)))

(defun read-single-float-array (array stream &key (start 0)
                                (end (array-total-size array)))
  (loop for i upfrom start below end do
    (setf (row-major-aref array i)
          (ieee-floats:decode-float32 (read-as-bytes 4 stream)))))

(defun read-double-float-array (array stream &key (start 0)
                                (end (array-total-size array)))
  (loop for i upfrom start below end do
    (setf (row-major-aref array i)
          (ieee-floats:decode-float64 (read-as-bytes 8 stream)))))



;;;; WITH-THREAD-CACHED-OBJECT

(defparameter *thread-caches* (tg:make-weak-hash-table :weakness :key))

(defvar *thread-cache-lock* (bordeaux-threads:make-lock "thread cache lock"))

(defun borrow-thread-cached-object (place-key key)
  (let ((thread-cache
          (bordeaux-threads:with-lock-held (*thread-cache-lock*)
            (gethash (bordeaux-threads:current-thread) *thread-caches*))))
    (when thread-cache<commit_msg>make *THREAD-CACHES* a DEFVAR<commit_after>(in-package :mgl-mat)

;;;; Float I/O

(defun write-as-bytes (integer n stream)
  (let ((x integer))
    (loop repeat n do
      (write-byte (logand x #xff) stream)
      (setq x (ash x -8)))
    (assert (zerop x))))

(defun read-as-bytes (n stream)
  (let ((x 0))
    (loop for i below n do
      (incf x (ash (read-byte stream) (* i 8))))
    x))

(defun write-single-float-array (array stream &key (start 0)
                                 (end (array-total-size array)))
  (loop for i upfrom start below end do
    (write-as-bytes (ieee-floats:encode-float32 (row-major-aref array i))
                    4 stream)))

(defun write-double-float-array (array stream &key (start 0)
                                 (end (array-total-size array)))
  (loop for i upfrom start below end do
    (write-as-bytes (ieee-floats:encode-float64 (row-major-aref array i))
                    8 stream)))

(defun read-single-float-array (array stream &key (start 0)
                                (end (array-total-size array)))
  (loop for i upfrom start below end do
    (setf (row-major-aref array i)
          (ieee-floats:decode-float32 (read-as-bytes 4 stream)))))

(defun read-double-float-array (array stream &key (start 0)
                                (end (array-total-size array)))
  (loop for i upfrom start below end do
    (setf (row-major-aref array i)
          (ieee-floats:decode-float64 (read-as-bytes 8 stream)))))



;;;; WITH-THREAD-CACHED-OBJECT

(defvar *thread-caches* (tg:make-weak-hash-table :weakness :key))

(defvar *thread-cache-lock* (bordeaux-threads:make-lock "thread cache lock"))

(defun borrow-thread-cached-object (place-key key)
  (let ((thread-cache
          (bordeaux-threads:with-lock-held (*thread-cache-lock*)
            (gethash (bordeaux-threads:current-thread) *thread-caches*))))
    (when thread-cache
{'fraction_non_alphanumeric': 0.14099840679766332, 'fraction_numerical': 0.01009028146574615, 'mean_word_length': 2.4028906955736224, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3570383', 'n_tokens_mistral': 1284, 'n_tokens_neox': 1240, 'n_words': 356}