<commit_before>(in-package :autowrap)

(defvar *foreign-bitmasks* (make-hash-table))


 ;; Masks

;;; Bitmasks aren't a type so we're not tying into foreign-type
(defun define-bitmask (name mask)
  (setf (gethash name *foreign-bitmasks*)
        (sort (copy-list mask) (lambda (a b) (< (cdr a) (cdr b)))))
  name)

(defun find-bitmask (name)
  (multiple-value-bind (mask exists-p)
      (gethash name *foreign-bitmasks*)
    (unless exists-p
      (error "Bitmask not found: ~S" name))
    mask))

(defun remove-bitmask (name)
  (remhash name *foreign-bitmasks*))

(defun mask-symbol-value (mask symbol)
  (or (aval symbol mask)
      (progn
        (warn "Unknown mask symbol ~S, treated as 0" symbol)
        0)))

(defun mask (name &rest symbols)
  "Create a mask by `LOGIOR` using the *list* of symbols `SYMBOLS`
from the bitmask `NAME`."
  (let ((mask (find-bitmask name)))
    (apply #'logior (mapcar (lambda (x) (mask-symbol-value mask x))
                            symbols))))

(define-compiler-macro mask (&whole whole name &rest symbols)
  (if (and (constantp name)
           (constantp symbols))
      (let ((symbols (eval symbols)))
        (let ((mask (find-bitmask (eval name))))
          (apply #'logior (mapcar (lambda (x) (mask-symbol-value mask x))
                                  symbols))))
      whole))

(defun mask-apply (name symbols)
  (apply #'mask name symbols))
<commit_msg>Allow bare numbers for bitmask values; also list valid symbols when incorrect symbol given<commit_after>(in-package :autowrap)

(defvar *foreign-bitmasks* (make-hash-table))


 ;; Masks

;;; Bitmasks aren't a type so we're not tying into foreign-type
(defun define-bitmask (name mask)
  (setf (gethash name *foreign-bitmasks*)
        (sort (copy-list mask) (lambda (a b) (< (cdr a) (cdr b)))))
  name)

(defun find-bitmask (name)
  (multiple-value-bind (mask exists-p)
      (gethash name *foreign-bitmasks*)
    (unless exists-p
      (error "Bitmask not found: ~S" name))
    mask))

(defun remove-bitmask (name)
  (remhash name *foreign-bitmasks*))

(defun mask-symbol-value (mask symbol)
  (etypecase symbol
    (integer symbol)
    (symbol
     (or (aval symbol mask)
         (progn
           (warn "Unknown mask symbol ~S, treated as 0; expected one of:~%  ~S"
                 symbol (sort (mapcar #'car mask) #'string<))
           0)))))

(defun mask (name &rest symbols)
  "Create a mask by `LOGIOR` using the *list* of symbols `SYMBOLS`
from the bitmask `NAME`."
  (let ((mask (find-bitmask name)))
    (apply #'logior (mapcar (lambda (x) (mask-symbol-value mask x))
                            symbols))))

(define-compiler-macro mask (&whole whole name &rest symbols)
  (if (and (constantp name)
           (constantp symbols))
      (let ((symbols (eval symbols)))
        (let ((mask (find-bitmask (eval name))))
          (apply #'logior (mapcar (lambda (x) (mask-symbol-value mask x))
                                  symbols))))
      whole))

(defun mask-apply (name symbols)
  (apply #'mask name symbols))

{'fraction_non_alphanumeric': 0.14055299539170507, 'fraction_numerical': 0.0013166556945358788, 'mean_word_length': 2.779850746268657, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1738584', 'n_tokens_mistral': 1020, 'n_tokens_neox': 993, 'n_words': 329}