<commit_before>(in-package #:spinneret)

(memoize
 (defun in? (element var)
   (not (null (find (string element)
                    (symbol-value var)
                    :key #'string
                    :test #'string=)))))

(defparameter *void-elements*
  '(!doctype area base br col command embed hr img
    input keygen link meta param source track wbr))

(defun void? (element)
  (in? element '*void-elements*))

(defparameter *literal-elements*
  '(pre script style))

(defun literal? (element)
  (in? element '*literal-elements*))

(defparameter *inline-elements*
  '(a abbr address bdo small code samp kbd
    cite strong dfn br em q data time var
    sub sup i b s u mark ruby rt rp bdi span wbr
    ins del col meter output))

(defun inline? (element)
  (values (in? element '*inline-elements*)))

(defparameter *paragraph-elements*
  '(meta title button label li h1 h2 h3 h4 h5 h6 p legend option
    dt dd figcaption iframe colgroup td th output summary command))

(defun paragraph? (element)
  (in? element '*paragraph-elements*))

(defparameter *end-tag-optional*
  ;; html head body
  '(li dt dd p rt rp optgroup
    option colgroup thead tbody tfoot tr td th))

(defun unmatched? (element)
  (in? element '*end-tag-optional*))

(defparameter *preformatted*
  '(pre textarea script style))

(defun preformatted? (element)
  (in? element '*preformatted*))

(defparameter *pseudotags*
  '((doctype . make-doctype)
    (!doctype . make-doctype)
    (cdata . make-cdata)
    (!-- . make-comment)
    (comment . make-comment)
    (html . make-html)
    (head . make-head)
    (raw . write-raw)))

(memoize
 (defun pseudotag-expander (element)
   (cdr (assoc (string element) *pseudotags*
               :test #'string= :key #'string))))

(defun pseudotag-expand (element args)
  (let ((expander (pseudotag-expander element)))
    (if expander
        (apply expander args)
        (cons element args))))<commit_msg>Meta doesn't need an end tag.<commit_after>(in-package #:spinneret)

(memoize
 (defun in? (element var)
   (not (null (find (string element)
                    (symbol-value var)
                    :key #'string
                    :test #'string=)))))

(defparameter *void-elements*
  '(!doctype area base br col command embed hr img
    input keygen link meta param source track wbr))

(defun void? (element)
  (in? element '*void-elements*))

(defparameter *literal-elements*
  '(pre script style))

(defun literal? (element)
  (in? element '*literal-elements*))

(defparameter *inline-elements*
  '(a abbr address bdo small code samp kbd
    cite strong dfn br em q data time var
    sub sup i b s u mark ruby rt rp bdi span wbr
    ins del col meter output))

(defun inline? (element)
  (values (in? element '*inline-elements*)))

(defparameter *paragraph-elements*
  '(meta title button label li h1 h2 h3 h4 h5 h6 p legend option
    dt dd figcaption iframe colgroup td th output summary command))

(defun paragraph? (element)
  (in? element '*paragraph-elements*))

(defparameter *end-tag-optional*
  ;; html head body
  '(li dt dd p rt rp optgroup
    option colgroup thead tbody tfoot tr td th
    meta))

(defun unmatched? (element)
  (in? element '*end-tag-optional*))

(defparameter *preformatted*
  '(pre textarea script style))

(defun preformatted? (element)
  (in? element '*preformatted*))

(defparameter *pseudotags*
  '((doctype . make-doctype)
    (!doctype . make-doctype)
    (cdata . make-cdata)
    (!-- . make-comment)
    (comment . make-comment)
    (html . make-html)
    (head . make-head)
    (raw . write-raw)))

(memoize
 (defun pseudotag-expander (element)
   (cdr (assoc (string element) *pseudotags*
               :test #'string= :key #'string))))

(defun pseudotag-expand (element args)
  (let ((expander (pseudotag-expander element)))
    (if expander
        (apply expander args)
        (cons element args))))
{'fraction_non_alphanumeric': 0.12580562000515597, 'fraction_numerical': 0.0030935808197989174, 'mean_word_length': 3.172043010752688, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5947487', 'n_tokens_mistral': 1325, 'n_tokens_neox': 1267, 'n_words': 433}