<commit_before>(defpackage :wookie-util
  (:use :cl :wookie-config :cl-async-future)
  (:export #:wlog
           #:map-plist
           #:camel-case
           #:querystringp
           #:set-querystring-hash
           #:querystring-to-hash
           #:body-to-string
           #:getf-reverse
           #:generate-tmp-file-name
           #:do-run-hooks
           #:lookup-status-text))
(in-package :wookie-util)

(defvar *tmp-file-counter* 0
  "Holds a value that is incremented for each temporary file generated.")

(defun wlog (level format-string &rest format-args)
  "Wookie's logging function. simple for now, just sends to STDOUT."
  (when (<= (getf +log-levels+ level) (getf +log-levels+ *log-level*))
    (let ((output (if *log-output*
                      *log-output*
                      t)))
      (apply #'format (append (list output format-string) format-args)))))
  
(defun map-plist (plist fn)
  "Iterate over a plist"
  (let ((result nil))
    (dotimes (i (/ (length plist) 2))
      (let (header value)
        (setf header (car plist)
              plist (cdr plist)
              value (car plist)
              plist (cdr plist))
        (let ((res (funcall fn header value)))
          (when res (push res result)))))
    (reverse result)))

(defun camel-case (keyword)
  "Camel case anything that can be converted to a string (string, keyword,
   symbol, etc)."
  (let* ((lower (string-downcase (string keyword))))
    (loop for i from 0
          for c across lower do
      (let ((char-before (when (< 0 i)
                           (aref lower (1- i)))))
        (when (or (= i 0)
                  (eq char-before #\-)
                  (eq char-before #\space)
                  (eq char-before #\tab))
          (setf (aref lower i) (aref (string-upcase (string c)) 0)))))
    lower))

(defparameter *scanner-querystring-p*
  (cl-ppcre:create-scanner "^([a-z-_\\[\\]]+(=[^&]+)?(&|$))+" :case-insensitive-mode t)
  "Detects a querystring.")

(defun querystringp (querystring)
  "Detects if the given string is an HTTP querystring."
  (cl-ppcre:scan *scanner-querystring-p* querystring))

(defun set-querystring-hash (hash key val)
  "Set the key of a querystring var into a hash, creating as many nested hashes
   as needed. For instance:
   
   (set-querystring-hash myhash \"data[body][tags]\" \"dogs,animals,bark\")
   
   Would update `myhash` with
     [body] => < hash: [tags] => \"dogs,animals,bark\" >"
  (let* ((subkeys (cl-ppcre:split "(\\]\\[|\\[|\\])" key))
         (subkeys (remove-if #'null subkeys))
         (curhash hash))
    (dotimes (i (length subkeys))
      (let ((subkey (car subkeys))
            (is-last (null (cdr subkeys))))
        (if is-last
            (progn
              (setf (gethash subkey curhash) val)
              (return))
            (progn
              (unless (hash-table-p (gethash subkey curhash))
                (setf (gethash subkey curhash) (make-hash-table :test #'equal)))
              (setf curhash (gethash subkey curhash)
                    subkeys (cdr subkeys))))))))

(defun querystring-to-hash (querystring)
  "Convert a querystring into a hash table."
  (let* ((main-hash (make-hash-table :test #'equal)))
    (unless querystring (return-from querystring-to-hash main-hash))
    (let* ((querystring (cl-ppcre:regex-replace-all "(^?|&$)" querystring ""))
           (parts (cl-ppcre:split "&(?!amp;)" querystring))
           (parts (remove-if #'null parts)))
      (dolist (part parts)
        (let* ((split (cl-ppcre:split "=" part))
               (key (car split))
               (val (cadr split))
               (val (do-urlencode:urldecode val :lenientp t :queryp t)))
          (set-querystring-hash main-hash key val)))
      main-hash)))

(defun print-hash (hash-table &optional (indent 0))
  "Useful for debugging hash tables."
  (loop for k being the hash-keys of hash-table
        for v being the hash-values of hash-table
        do
    (dotimes (i indent) (format t " "))
    (if (hash-table-p v)
        (progn
          (format t "[~a] => ~%" k)
          (print-hash v (+ 2 indent)))
    (format t "[~a] => ~s~%" k v))))

(defun body-to-string (body-bytes content-type-header)
  "Given a byte vector of HTTP body data and the value of a Content-Type header,
   convert the body to a string via the charset provided in the header. If a
   character encoding is not provided, go with the HTTP default, ISO-8859-1."
  (let* ((charset-pos (search "charset=" content-type-header))
         (charset (if charset-pos
                      (intern (string-upcase (subseq content-type-header
                                                     (+ charset-pos 8))) :keyword)
                      :iso-8859-1)))
    (handler-case
      (babel:octets-to-string body-bytes :encoding charset)
      (t ()
        (babel:octets-to-string body-bytes :encoding :iso-8859-1))))) 

(defun getf-reverse (plist key)
  "Like getf, except the VALUE comes before the KEY:
     '(:value1 :key1 :value2 :key2)
   Allows reverse lookups in plists without duplicating structures."
  (dotimes (i (length plist))
    (when (eq key (cadr plist))
      (return-from getf-reverse (car plist)))
    (setf plist (cddr plist))))

(defun generate-tmp-file-name ()
  "Generate the a full path/filename for a temporary file that does not exist
   already in the tmp directory."
  (format nil "~atmp~a" (namestring *tmp-file-store*) (incf *tmp-file-counter*)))

(defmacro do-run-hooks ((socket) run-hook-cmd &body body)
  "Run a number of hooks, catch any errors while running said hooks, and if an
   error occurs, clear out all traces of the current request (specified on the
   socket). If no errors occur, run the body normally."
  (let ((sock (gensym "sock")))
    `(let ((,sock ,socket))
       (future-handler-case
         (wait-for ,run-hook-cmd
           ,@body)
         (error ()
           (if (as:socket-closed-p ,sock)
               ;; reset the parser for this socket if it's open. this
               ;; should suffice as far as garbage collection goes.
               (setup-parser ,sock)
               ;; clear out the socket's data, just in case
               (setf (as:socket-data ,sock) nil)))))))

(defun lookup-status-text (status-code)
  "Get the HTTP standard text that goes along with a status code."
  (case status-code
    (100 "Continue")
    (101 "Switching Protocols")
    (102 "Processing")

    (200 "OK")
    (201 "Created")
    (202 "Accepted")
    (203 "Non-Authoratative Information")
    (204 "No Content")<commit_msg>oh god...derrr....   fixing bug in new hook error handling.<commit_after>(defpackage :wookie-util
  (:use :cl :wookie-config :cl-async-future)
  (:export #:wlog
           #:map-plist
           #:camel-case
           #:querystringp
           #:set-querystring-hash
           #:querystring-to-hash
           #:body-to-string
           #:getf-reverse
           #:generate-tmp-file-name
           #:do-run-hooks
           #:lookup-status-text))
(in-package :wookie-util)

(defvar *tmp-file-counter* 0
  "Holds a value that is incremented for each temporary file generated.")

(defun wlog (level format-string &rest format-args)
  "Wookie's logging function. simple for now, just sends to STDOUT."
  (when (<= (getf +log-levels+ level) (getf +log-levels+ *log-level*))
    (let ((output (if *log-output*
                      *log-output*
                      t)))
      (apply #'format (append (list output format-string) format-args)))))
  
(defun map-plist (plist fn)
  "Iterate over a plist"
  (let ((result nil))
    (dotimes (i (/ (length plist) 2))
      (let (header value)
        (setf header (car plist)
              plist (cdr plist)
              value (car plist)
              plist (cdr plist))
        (let ((res (funcall fn header value)))
          (when res (push res result)))))
    (reverse result)))

(defun camel-case (keyword)
  "Camel case anything that can be converted to a string (string, keyword,
   symbol, etc)."
  (let* ((lower (string-downcase (string keyword))))
    (loop for i from 0
          for c across lower do
      (let ((char-before (when (< 0 i)
                           (aref lower (1- i)))))
        (when (or (= i 0)
                  (eq char-before #\-)
                  (eq char-before #\space)
                  (eq char-before #\tab))
          (setf (aref lower i) (aref (string-upcase (string c)) 0)))))
    lower))

(defparameter *scanner-querystring-p*
  (cl-ppcre:create-scanner "^([a-z-_\\[\\]]+(=[^&]+)?(&|$))+" :case-insensitive-mode t)
  "Detects a querystring.")

(defun querystringp (querystring)
  "Detects if the given string is an HTTP querystring."
  (cl-ppcre:scan *scanner-querystring-p* querystring))

(defun set-querystring-hash (hash key val)
  "Set the key of a querystring var into a hash, creating as many nested hashes
   as needed. For instance:
   
   (set-querystring-hash myhash \"data[body][tags]\" \"dogs,animals,bark\")
   
   Would update `myhash` with
     [body] => < hash: [tags] => \"dogs,animals,bark\" >"
  (let* ((subkeys (cl-ppcre:split "(\\]\\[|\\[|\\])" key))
         (subkeys (remove-if #'null subkeys))
         (curhash hash))
    (dotimes (i (length subkeys))
      (let ((subkey (car subkeys))
            (is-last (null (cdr subkeys))))
        (if is-last
            (progn
              (setf (gethash subkey curhash) val)
              (return))
            (progn
              (unless (hash-table-p (gethash subkey curhash))
                (setf (gethash subkey curhash) (make-hash-table :test #'equal)))
              (setf curhash (gethash subkey curhash)
                    subkeys (cdr subkeys))))))))

(defun querystring-to-hash (querystring)
  "Convert a querystring into a hash table."
  (let* ((main-hash (make-hash-table :test #'equal)))
    (unless querystring (return-from querystring-to-hash main-hash))
    (let* ((querystring (cl-ppcre:regex-replace-all "(^?|&$)" querystring ""))
           (parts (cl-ppcre:split "&(?!amp;)" querystring))
           (parts (remove-if #'null parts)))
      (dolist (part parts)
        (let* ((split (cl-ppcre:split "=" part))
               (key (car split))
               (val (cadr split))
               (val (do-urlencode:urldecode val :lenientp t :queryp t)))
          (set-querystring-hash main-hash key val)))
      main-hash)))

(defun print-hash (hash-table &optional (indent 0))
  "Useful for debugging hash tables."
  (loop for k being the hash-keys of hash-table
        for v being the hash-values of hash-table
        do
    (dotimes (i indent) (format t " "))
    (if (hash-table-p v)
        (progn
          (format t "[~a] => ~%" k)
          (print-hash v (+ 2 indent)))
    (format t "[~a] => ~s~%" k v))))

(defun body-to-string (body-bytes content-type-header)
  "Given a byte vector of HTTP body data and the value of a Content-Type header,
   convert the body to a string via the charset provided in the header. If a
   character encoding is not provided, go with the HTTP default, ISO-8859-1."
  (let* ((charset-pos (search "charset=" content-type-header))
         (charset (if charset-pos
                      (intern (string-upcase (subseq content-type-header
                                                     (+ charset-pos 8))) :keyword)
                      :iso-8859-1)))
    (handler-case
      (babel:octets-to-string body-bytes :encoding charset)
      (t ()
        (babel:octets-to-string body-bytes :encoding :iso-8859-1))))) 

(defun getf-reverse (plist key)
  "Like getf, except the VALUE comes before the KEY:
     '(:value1 :key1 :value2 :key2)
   Allows reverse lookups in plists without duplicating structures."
  (dotimes (i (length plist))
    (when (eq key (cadr plist))
      (return-from getf-reverse (car plist)))
    (setf plist (cddr plist))))

(defun generate-tmp-file-name ()
  "Generate the a full path/filename for a temporary file that does not exist
   already in the tmp directory."
  (format nil "~atmp~a" (namestring *tmp-file-store*) (incf *tmp-file-counter*)))

(defmacro do-run-hooks ((socket) run-hook-cmd &body body)
  "Run a number of hooks, catch any errors while running said hooks, and if an
   error occurs, clear out all traces of the current request (specified on the
   socket). If no errors occur, run the body normally."
  (let ((sock (gensym "sock")))
    `(let ((,sock ,socket))
       (future-handler-case
         (wait-for ,run-hook-cmd
           ,@body)
         (error ()
           (if (as:socket-closed-p ,sock)
               ;; clear out the socket's data, just in case
               (setf (as:socket-data ,sock) nil)
               ;; reset the parser for this socket if it's open. this
               ;; should suffice as far as garbage collection goes.
               (setup-parser ,sock)))))))

(defun lookup-status-text (status-code)
  "Get the HTTP standard text that goes along with a status code."
  (case status-code
    (100 "Continue")
    (101 "Switching Protocols")
    (102 "Processing")

    (200 "OK")
    (201 "Created")
    (202 "Accepted")
    (203 "Non-Authoratative Information")
    (204 "No Content")
{'fraction_non_alphanumeric': 0.14083434835566383, 'fraction_numerical': 0.008069427527405602, 'mean_word_length': 2.2703510082150857, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '698498', 'n_tokens_mistral': 4329, 'n_tokens_neox': 4084, 'n_words': 1333}