<commit_before>#lang scribble/doc

@(begin

(require (for-syntax scheme/base
                     syntax/boundmap
                     scheme/list
                     compiler/cm-accomplice)
         scribble/manual
         scribble/struct
         scribble/basic
         scribble/decode
         scheme/include)

;; define `chunk' as a macro that typesets the code
(define-syntax (chunk stx)
  (syntax-case stx ()
    [(_ name expr ...)
     (cond
       [(not (identifier? #'name))
        (raise-syntax-error #f "expected a chunk name" stx #'name)]
       [(not (regexp-match? #rx"^<.*>$" (symbol->string (syntax-e #'name))))
        (raise-syntax-error
         #f "chunk names must begin and end with angle brackets, <...>"
         stx #'name)]
       [else #`(make-splice (list (emph (scheme name) " ::=")
                                  (schemeblock expr ...)))])]))

(define-syntax module
  (syntax-rules () [(module name base body ...) (begin body ...)]))
(include "chat-noir-literate.ss")

)
<commit_msg>* There's no need to do the syntactic checks of identifiers for   `chunk', if there's an error, then people will get it anyway when   they run the code.  This means that `chunk' can be a simple   syntax-rule. * Removed some redundant requires.<commit_after>#lang scribble/doc

@(begin

(require scribble/manual scribble/decode scheme/include)

;; define `chunk' as a macro that typesets the code
(define-syntax-rule (chunk name expr ...)
  (make-splice (list (emph (scheme name) " ::=")
                     (schemeblock expr ...))))

(define-syntax module
  (syntax-rules () [(module name base body ...) (begin body ...)]))
(include "chat-noir-literate.ss")

)

{'fraction_non_alphanumeric': 0.14328180737217597, 'fraction_numerical': 0.0, 'mean_word_length': 2.3392857142857144, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '660923', 'n_tokens_mistral': 536, 'n_tokens_neox': 492, 'n_words': 158}