<commit_before>;;;; dishes.lisp

(in-package #:dishes)

(defvar *print-commented-forms* nil
  "Print what when a comment-line-supress-forms reader macro is triggerd?
  
  Options are :COMMENT or T     -- the comment line
              NIL or :NONE      -- don't print
              :FORM             -- bind *READ-SUPRESS* to nil, print read forms.")

(defun comment-line-suppress-forms (stream char &optional count)
  "Comments the rest of the line then read-suppresses COUNT forms, or 1 form.

Rationale: #+(or) is seen by many to be cryptic, and #+(or) is more
useful when accompanied by an explanatory comment."
  (declare (ignorable char))
  (cond ((member *print-commented-forms* '(t :comment :form))
	 (format t "~&~%#~@[~A~]~A~A" count char (read-line stream t nil t)))
	(t (do () ((char= (read-char stream nil #\Newline t) #\Newline)))))
  (cond ((eq *print-commented-forms* :form)
	 (let ((*read-suppress* nil))
	   (loop repeat (or count 1) do (print (read stream t nil t)))
	   (values)))
	(t (let ((*read-suppress* t))
	     (loop repeat (or count 1) do (read stream t nil t))
	     (values)))))

(defun pathname-this-directory-reader (stream char &optional count)
  "Read form and return a pathname in the files directory.

Assumes the form is a backslash escaped form ending at the first
unescaped (closer char)."
  (declare (ignore count))
  (let ((end-char (closer char)))
    (merge-pathnames
     (with-output-to-string (name.type)
       (loop for c = (read-char stream t nil t)
             until (char= c end-char)
             do (write-char (if (char= c #\\)
                                (read-char stream t nil t)
                                c)
                            name.type)))
     (or *compile-file-truename*
         *load-truename*
         *default-pathname-defaults*))))

(defun run-time-symbol-reader (stream char &optional count)
  "Reads a delimited list into a form that will find-symbol at runtime."
  (declare (ignore count))
  (destructuring-bind (package name) (read-delimited-list (closer char) stream)
    `(find-symbol ,(symbol-name name) ,(symbol-name package))))

(defun make-interned-string-reader (&key case-insensitive-p)
  (let ((words (make-hash-table :test (if case-insensitive-p 'equalp 'equal))))
    (lambda (stream char &optional count)
      (declare (ignore count))
      (if stream
          (let* ((string (with-output-to-string (s)
                           (loop for c = (read-char stream t)
                                 until (char= c (closer char))<commit_msg>pathname-this-directory-reader: don't default type<commit_after>;;;; dishes.lisp

(in-package #:dishes)

(defvar *print-commented-forms* nil
  "Print what when a comment-line-supress-forms reader macro is triggerd?
  
  Options are :COMMENT or T     -- the comment line
              NIL or :NONE      -- don't print
              :FORM             -- bind *READ-SUPRESS* to nil, print read forms.")

(defun comment-line-suppress-forms (stream char &optional count)
  "Comments the rest of the line then read-suppresses COUNT forms, or 1 form.

Rationale: #+(or) is seen by many to be cryptic, and #+(or) is more
useful when accompanied by an explanatory comment."
  (declare (ignorable char))
  (cond ((member *print-commented-forms* '(t :comment :form))
	 (format t "~&~%#~@[~A~]~A~A" count char (read-line stream t nil t)))
	(t (do () ((char= (read-char stream nil #\Newline t) #\Newline)))))
  (cond ((eq *print-commented-forms* :form)
	 (let ((*read-suppress* nil))
	   (loop repeat (or count 1) do (print (read stream t nil t)))
	   (values)))
	(t (let ((*read-suppress* t))
	     (loop repeat (or count 1) do (read stream t nil t))
	     (values)))))

(defun pathname-this-directory-reader (stream char &optional count)
  "Read form and return a pathname in the files directory.

Assumes the form is a backslash escaped form ending at the first
unescaped (closer char)."
  (declare (ignore count))
  (let ((end-char (closer char)))
    (merge-pathnames
     (with-output-to-string (name.type)
       (loop for c = (read-char stream t nil t)
             until (char= c end-char)
             do (write-char (if (char= c #\\)
                                (read-char stream t nil t)
                                c)
                            name.type)))
     (make-pathname :directory
                    (pathname-directory
                     (or *compile-file-truename*
                         *load-truename*
                         *default-pathname-defaults*))))))

(defun run-time-symbol-reader (stream char &optional count)
  "Reads a delimited list into a form that will find-symbol at runtime."
  (declare (ignore count))
  (destructuring-bind (package name) (read-delimited-list (closer char) stream)
    `(find-symbol ,(symbol-name name) ,(symbol-name package))))

(defun make-interned-string-reader (&key case-insensitive-p)
  (let ((words (make-hash-table :test (if case-insensitive-p 'equalp 'equal))))
    (lambda (stream char &optional count)
      (declare (ignore count))
      (if stream
          (let* ((string (with-output-to-string (s)
                           (loop for c = (read-char stream t)
                                 until (char= c (closer char))
{'fraction_non_alphanumeric': 0.1316291491797024, 'fraction_numerical': 0.0011446012972148034, 'mean_word_length': 2.4290385873119686, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1715379', 'n_tokens_mistral': 1653, 'n_tokens_neox': 1597, 'n_words': 563}