<commit_before>;; -*- lisp -*-

(in-package :it.bese.arnesi)

;;;; * HTTP/HTML utilities

;;;; ** URIs/URLs

(eval-always
  (defvar *uri-escaping-ok-table* (make-array 256
                                              :element-type 'boolean
                                              :initial-element nil))
  (loop
      ;; The list of characters which don't need to be escaped when writing URIs.
      for ok-char across "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.," do
      (setf (aref *uri-escaping-ok-table* (char-code ok-char)) t))
  (setf *uri-escaping-ok-table* (coerce *uri-escaping-ok-table* '(simple-array boolean (256)))))

(defun escape-as-uri (string)
  "Escapes all non alphanumeric characters in STRING following
  the URI convention. Returns a fresh string."
  (with-output-to-string (escaped)
    (write-as-uri string escaped)))

(defun write-as-uri (string stream)
  (declare (type vector string)
           (type stream stream)
           (optimize (speed 3) (debug 0)))
  (loop
      for char-code :of-type (unsigned-byte 8) :across (the (vector (unsigned-byte 8))
                                                         (string-to-octets string :utf-8)) do
      (if (aref (the (simple-array boolean (256)) (load-time-value *uri-escaping-ok-table* t)) char-code)
          (write-char (code-char char-code) stream)
          (format stream "%~2,'0X" char-code))))

(define-condition uri-parse-error (error)
  ((what :initarg :what :reader uri-parse-error.what)))

(define-condition expected-digit-uri-parse-error (uri-parse-error) ())

(defun continue-as-is (c)
  (declare (ignore c))
  (awhen (find-restart 'continue-as-is)
    (invoke-restart it)))

(defun unescape-as-uri-non-strict (string)
  (handler-bind ((expected-digit-uri-parse-error #'continue-as-is))
    (unescape-as-uri string)))

(defun %unescape-as-uri (input)
  "URI unescape based on http://www.ietf.org/rfc/rfc2396.txt"
  (declare (type string input)
           (optimize (speed 3) (debug 0)))
  (let ((input-length (length input)))
    (when (zerop input-length)
      (return-from %unescape-as-uri ""))
    (let* ((input-index 0)
           (output (make-array input-length :element-type '(unsigned-byte 8) :adjustable t :fill-pointer 0)))
      (declare (type fixnum input-length input-index))
      (labels ((read-next-char (must-exists-p)
                 (when (>= input-index input-length)
                   (if must-exists-p
                       (error 'uri-parse-error :what input)
                       (return-from %unescape-as-uri (octets-to-string output :utf-8))))
                 (prog1 (aref input input-index)
                   (incf input-index)))
               (write-next-byte (byte)
                 (vector-push-extend  byte output))
               (char-to-int (char)
                 (let ((result (digit-char-p char 16)))
                   (unless result
                     (error 'expected-digit-uri-parse-error :what char))
                   result))
               (parse ()
                 (let ((next-char (read-next-char nil)))
                   (case next-char
                     (#\% (char%))<commit_msg>Small change in unescape-as-uri to muffle some compiler notes<commit_after>;; -*- lisp -*-

(in-package :it.bese.arnesi)

;;;; * HTTP/HTML utilities

;;;; ** URIs/URLs

(eval-always
  (defvar *uri-escaping-ok-table* (make-array 256
                                              :element-type 'boolean
                                              :initial-element nil))
  (loop
      ;; The list of characters which don't need to be escaped when writing URIs.
      for ok-char across "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.," do
      (setf (aref *uri-escaping-ok-table* (char-code ok-char)) t))
  (setf *uri-escaping-ok-table* (coerce *uri-escaping-ok-table* '(simple-array boolean (256)))))

(defun escape-as-uri (string)
  "Escapes all non alphanumeric characters in STRING following
  the URI convention. Returns a fresh string."
  (with-output-to-string (escaped)
    (write-as-uri string escaped)))

(defun write-as-uri (string stream)
  (declare (type vector string)
           (type stream stream)
           (optimize (speed 3) (debug 0)))
  (loop
      for char-code :of-type (unsigned-byte 8) :across (the (vector (unsigned-byte 8))
                                                         (string-to-octets string :utf-8)) do
      (if (aref (the (simple-array boolean (256)) (load-time-value *uri-escaping-ok-table* t)) char-code)
          (write-char (code-char char-code) stream)
          (format stream "%~2,'0X" char-code))))

(define-condition uri-parse-error (error)
  ((what :initarg :what :reader uri-parse-error.what)))

(define-condition expected-digit-uri-parse-error (uri-parse-error) ())

(defun continue-as-is (c)
  (declare (ignore c))
  (awhen (find-restart 'continue-as-is)
    (invoke-restart it)))

(defun unescape-as-uri-non-strict (string)
  (handler-bind ((expected-digit-uri-parse-error #'continue-as-is))
    (unescape-as-uri string)))

(defun %unescape-as-uri (input)
  "URI unescape based on http://www.ietf.org/rfc/rfc2396.txt"
  (declare (type string input)
           (optimize (speed 3) (debug 0)))
  (let ((input-length (length input)))
    (when (zerop input-length)
      (return-from %unescape-as-uri ""))
    (let* ((input-index 0)
           (output (make-array input-length :element-type '(unsigned-byte 8) :adjustable t :fill-pointer 0)))
      (declare (type fixnum input-length input-index))
      (labels ((read-next-char (must-exists-p)
                 (when (>= input-index input-length)
                   (if must-exists-p
                       (error 'uri-parse-error :what input)
                       (return-from %unescape-as-uri (octets-to-string output :utf-8))))
                 (prog1 (aref input input-index)
                   (incf input-index)))
               (write-next-byte (byte)
                 (vector-push-extend byte output)
                 (values))
               (char-to-int (char)
                 (let ((result (digit-char-p char 16)))
                   (unless result
                     (error 'expected-digit-uri-parse-error :what char))
                   result))
               (parse ()
                 (let ((next-char (read-next-char nil)))
                   (case next-char
                     (#\% (char%))
{'fraction_non_alphanumeric': 0.14105164612264004, 'fraction_numerical': 0.012170385395537525, 'mean_word_length': 2.0407969639468693, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '861098', 'n_tokens_mistral': 2089, 'n_tokens_neox': 1968, 'n_words': 496}