<commit_before>(in-package :http-parse)

(deftype simple-byte-vector (&optional (len '*))
  `(simple-array (unsigned-byte 8) (,len)))

(defun append-array (arr1 arr2)
  "Create an array, made up of arr1 followed by arr2."
  (let ((arr1-length (length arr1))
        (arr2-length (length arr2)))
    (let ((arr (make-array (+ arr1-length arr2-length)
                           :element-type (array-element-type arr1))))
      (replace arr arr1 :start1 0)
      (replace arr arr2 :start1 arr1-length)
      arr)))

(defun find-non-whitespace-pos (seq &key (start 0))
  "Find the position of the first non-whitespace character in an octet sequence."
  (declare (type simple-byte-vector seq)
           (optimize (speed 3) (safety 0)))
  (let ((length (length seq)))
    (unless (zerop length)
      (do* ((i start (1+ i))
            (byte (aref seq i) (aref seq i)))
           ((= i length))
        (unless (or (= byte 9)
                    (= byte 13)
                    (= byte 10)
                    (= byte 32))
          (return-from find-non-whitespace-pos i))))))

(declaim (ftype (function ((unsigned-byte 8)) (unsigned-byte 8)) byte-to-ascii-upper)
         (inline byte-to-ascii-upper))
(defun byte-to-ascii-upper (x)
  (declare (type (unsigned-byte 8) x)
           (optimize (speed 3) (safety 0)))
  (if (>= #.(char-code #\z) x #.(char-code #\a))
      (+ #.(- (char-code #\A) (char-code #\a)) x)
      x))

(defun ascii-octets-to-upper-string (octets)
  (declare (type simple-byte-vector octets)
           (optimize (speed 3) (safety 0)))
  (let* ((len (length octets))
         (string (make-string len :element-type 'character)))
    (dotimes (i len string)
      (setf (aref string i)
            (code-char (byte-to-ascii-upper (aref octets i)))))))<commit_msg>use logical or for upper-case<commit_after>(in-package :http-parse)

(deftype simple-byte-vector (&optional (len '*))
  `(simple-array (unsigned-byte 8) (,len)))

(defun append-array (arr1 arr2)
  "Create an array, made up of arr1 followed by arr2."
  (let ((arr1-length (length arr1))
        (arr2-length (length arr2)))
    (let ((arr (make-array (+ arr1-length arr2-length)
                           :element-type (array-element-type arr1))))
      (replace arr arr1 :start1 0)
      (replace arr arr2 :start1 arr1-length)
      arr)))

(defun find-non-whitespace-pos (seq &key (start 0))
  "Find the position of the first non-whitespace character in an octet sequence."
  (declare (type simple-byte-vector seq)
           (optimize (speed 3) (safety 0)))
  (let ((length (length seq)))
    (unless (zerop length)
      (do* ((i start (1+ i))
            (byte (aref seq i) (aref seq i)))
           ((= i length))
        (unless (or (= byte 9)
                    (= byte 13)
                    (= byte 10)
                    (= byte 32))
          (return-from find-non-whitespace-pos i))))))

(declaim (ftype (function ((unsigned-byte 8)) (unsigned-byte 8)) byte-to-ascii-upper)
         (inline byte-to-ascii-upper))
(defun byte-to-ascii-upper (x)
  (declare (type (unsigned-byte 8) x)
           (optimize (speed 3) (safety 0)))
  (logior x (the (unsigned-byte 8) #.(ash 1 5))))

(defun ascii-octets-to-upper-string (octets)
  (declare (type simple-byte-vector octets)
           (optimize (speed 3) (safety 0)))
  (let* ((len (length octets))
         (string (make-string len :element-type 'character)))
    (dotimes (i len string)
      (setf (aref string i)
            (code-char (byte-to-ascii-upper (aref octets i)))))))
{'fraction_non_alphanumeric': 0.16009083167754753, 'fraction_numerical': 0.02128867442520579, 'mean_word_length': 2.39172281039461, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '698497', 'n_tokens_mistral': 1259, 'n_tokens_neox': 1177, 'n_words': 359}