<commit_before>;; -*- lisp -*-

(in-package :it.bese.arnesi)

;;;; * HTTP/HTML utilities

;;;; ** URIs/URLs

(defvar *ok-set*
  "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.,"
  "The list of characters which don't need to be escaped when
  writing URIs.")

(defun escape-as-uri (string)
  "Escapes all non alphanumeric characters in STRING following
  the URI convention. Returns a fresh string."
  (with-output-to-string (escaped)
    (write-as-uri string escaped)))

(defun write-as-uri (string stream)
  (loop
     for char across string
     if (find char *ok-set* :test #'char=)
       do (write-char char stream)
     else do (format stream "%~2,'0X" (char-code char))))

(defun make-escaped-table ()
  (let ((table (make-array '(16 16)
                           :element-type 'character
                           :initial-element #\\)))
    (dotimes (i 16)
      (dotimes (j 16)
        (setf (aref table i j) (code-char (+ (* i 16) j)))))
    table))

(defvar *unescape-table* (make-escaped-table))

(defun nunescape-as-uri (string &rest args)
  (apply #'unescape-as-uri string args))

(defun unescape-as-uri (string &optional (external-format :iso-8859-1))
  (let* ((length (- (length string)
                    (* 2 (count #\% string :test #'char=))))
         (result (make-array length :element-type '(unsigned-byte 8))))
    (loop
       for index1 upfrom 0
       for index2 upfrom 0
       while (< index1 (length string))
       do (setf (aref result index2)
                (case (aref string index1)
                  (#\% (+ (ash (digit-char-p (aref string (incf index1)) 16) 4)
                          (digit-char-p (aref string (incf index1)) 16)))
                  (#\+ #.(char-code #\space))
                  (t (char-code (aref string index1))))))
    (octets-to-string result external-format))) 

;;;; ** HTML<commit_msg>Fix unescape-as-uri and optimize it while we are at it<commit_after>;; -*- lisp -*-

(in-package :it.bese.arnesi)

;;;; * HTTP/HTML utilities

;;;; ** URIs/URLs

(defvar *ok-set*
  "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.,"
  "The list of characters which don't need to be escaped when
  writing URIs.")

(defun escape-as-uri (string)
  "Escapes all non alphanumeric characters in STRING following
  the URI convention. Returns a fresh string."
  (with-output-to-string (escaped)
    (write-as-uri string escaped)))

(defun write-as-uri (string stream)
  (loop
     for char across string
     if (find char *ok-set* :test #'char=)
       do (write-char char stream)
     else do (format stream "%~2,'0X" (char-code char))))

(defun make-escaped-table ()
  (let ((table (make-array '(16 16)
                           :element-type 'character
                           :initial-element #\\)))
    (dotimes (i 16)
      (dotimes (j 16)
        (setf (aref table i j) (code-char (+ (* i 16) j)))))
    table))

(defvar *unescape-table* (make-escaped-table))

(defun unescape-as-uri (string)
  (declare (inline))
  (%unescape-as-uri string nil))

(defun nunescape-as-uri (string)
  (declare (inline))
  (%unescape-as-uri string t))

(defun %unescape-as-uri (input may-modify-input-p)
  (declare (type vector input)
           (optimize (speed 3) (debug 0)))
  (let* ((input-length (length input))
         (input-index 0)
         (output (if may-modify-input-p
                     input
                     (make-array input-length :element-type 'character :adjustable t)))
         (output-index 0))
    (declare (type fixnum input-index output-index))
    (labels ((fail ()
               (error "Parse error in unescape-as-uri for string ~S" input))
             (read-next-char (&optional must-exists-p)
               (when (>= input-index input-length)
                 (if must-exists-p
                     (fail)
                     (return-from %unescape-as-uri (adjust-array output output-index))))
               (prog1 (aref input input-index)
                 (incf input-index)))
             (write-next-char (char)
               (setf (aref output output-index) char)
               (incf output-index))
             (char-to-int (char)
               (let ((result (digit-char-p char 16)))
                 (unless result
                   (fail))
                 result))
             (parse ()
               (let ((next-char (read-next-char)))
                 (case next-char
                   (#\% (char%))
                   (#\+ (char+))
                   (t (write-next-char next-char)))
                 (parse)))
             (char% ()
               (let ((next-char (read-next-char t)))
                 (write-next-char (code-char (case next-char
                                               (#\u (+ (ash (char-to-int (read-next-char t)) 12)
                                                       (ash (char-to-int (read-next-char t)) 8)
                                                       (ash (char-to-int (read-next-char t)) 4)
                                                       (char-to-int (read-next-char t))))
                                               (t (+ (ash (char-to-int next-char) 4)
                                                     (char-to-int (read-next-char t)))))))))
             (char+ ()
               (write-next-char #\space)))
      (parse))))

;;;; ** HTML
{'fraction_non_alphanumeric': 0.14360704911886013, 'fraction_numerical': 0.014623172103487065, 'mean_word_length': 1.706747843734145, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '53396', 'n_tokens_mistral': 1720, 'n_tokens_neox': 1619, 'n_words': 443}