<commit_before>
(define (vaquero-write obj port)
   (write (vaquero-view obj) port))

(define (vaquero-print obj port)
   (display (vaquero-view obj) port))

(define (vaquero-read port)
   (define first-rune (peek-char port))
   (if (eof-object? first-rune)
      first-rune
      (vaquero-parse (vaquero-reader port))))

(define (vaquero-reader port)
   (define token (peek-char port))
   (if (char-whitespace? token)
      (let ((_ (read-char port)) (next (peek-char port)))
         (if (eof-object? next)
            next
            (vaquero-reader port)))
      (case token
         ((#\()
            (let ((t (read-char port)) (peek-a-boo (peek-char port)))
               (if (eq? peek-a-boo #\))
                  (begin
                     (read-char port)
                     '())
                  (vaquero-read-list port))))
         ((#\)) (vaquero-error 'read-error 'read "read error: unexpected ')'"))
         ((#\') (vaquero-read-quote port))
         ((#\%) (vaquero-read-quasiquote port))
         ((#\$) (vaquero-read-unquote port))
         ((#\@) (vaquero-read-unquote-splicing port))
         ((#\#) (vaquero-read-structure port))
         ((#\;) (vaquero-read-comment port) (vaquero-read port))
         ((#\| #\\) (vaquero-read-funky port))
         (else (read port)))))

(define (vaquero-read-error-handler e kont)
   (debug 'read-error
      (if (vaquero-error? e)
         (map (lambda (f) (vaquero-view (vaquero-send-atomic e f))) '(name form to-text))
         (vaquero-view e)))
   (exit))

(define (vaquero-read-structure port)
   (read-char port) ; ditch the #
   (read-char port) ; ditch the (
   (let ((type (read port))) ; read the symbol in head
      (if (not (symbol? type))
         (vaquero-error 'read-error type "read error: structures must begin with a type symbol!")
         (if (holy? type)
            (let ((funk (glookup type)) (args (vaquero-read-list port)))
               (vaquero-apply funk args my-empty-table identity vaquero-read-error-handler))
            (case type
               ((text)    (vaquero-read-text port))
               ((template) (vaquero-read-template port))
               ((doc)     (vaquero-read-doc port) (vaquero-reader port))
               (else      (vaquero-error 'unknown-structure type (string-join (list "read error: unknown structure type:" (symbol->string type) "is not defined in the global environment.") " "))))))))

(define (vaquero-read-list port)
   ; lparen already swallowed
   (let loop ((token (peek-char port)) (acc '()))
      (cond
       ((eof-object? token)
          (vaquero-error 'read-error 'EOF "read error: unexpected EOF in unterminated list!"))
       ((char-whitespace? token)
          (read-char port)
          (loop (peek-char port) acc))<commit_msg>Hack reader to handle R7RS keywords... firk ding blast<commit_after>
(define (vaquero-write obj port)
   (write (vaquero-view obj) port))

(define (vaquero-print obj port)
   (display (vaquero-view obj) port))

(define (vaquero-read port)
   (define first-rune (peek-char port))
   (if (eof-object? first-rune)
      first-rune
      (vaquero-parse (vaquero-reader port))))

(define (vaquero-reader port)
   (define token (peek-char port))
   (if (char-whitespace? token)
      (let ((_ (read-char port)) (next (peek-char port)))
         (if (eof-object? next)
            next
            (vaquero-reader port)))
      (case token
         ((#\()
            (let ((t (read-char port)) (peek-a-boo (peek-char port)))
               (if (eq? peek-a-boo #\))
                  (begin
                     (read-char port)
                     '())
                  (vaquero-read-list port))))
         ((#\)) (vaquero-error 'read-error 'read "read error: unexpected ')'"))
         ((#\') (vaquero-read-quote port))
         ((#\%) (vaquero-read-quasiquote port))
         ((#\$) (vaquero-read-unquote port))
         ((#\@) (vaquero-read-unquote-splicing port))
         ((#\#) (vaquero-read-structure port))
         ((#\;) (vaquero-read-comment port) (vaquero-read port))
         ((#\| #\\) (vaquero-read-funky port))
         (else (read port)))))

(define (vaquero-read-error-handler e kont)
   (debug 'read-error
      (if (vaquero-error? e)
         (map (lambda (f) (vaquero-view (vaquero-send-atomic e f))) '(name form to-text))
         (vaquero-view e)))
   (exit))

(define (vaquero-read-structure port)
   (let ((nothing (read-char port)) ; ditch the #
         (label (read-char port)))  ; either : for chicken keyword or ( for vaquero structure
      (if (eq? label #\:)
         (let ((sym (read port)))
            (symbol->keyword sym))
         (let ((type (read port))) ; read the symbol in head
            (if (not (symbol? type))
               (vaquero-error 'read-error type "read error: structures must begin with a type symbol!")
               (if (holy? type)
                  (let ((funk (glookup type)) (args (vaquero-read-list port)))
                     (vaquero-apply funk args my-empty-table identity vaquero-read-error-handler))
                  (case type
                     ((text)     (vaquero-read-text port))
                     ((template) (vaquero-read-template port))
                     ((doc)      (vaquero-read-doc port) (vaquero-reader port))
                     (else       (vaquero-error 'unknown-structure type
                                    (string-join (list "read error: unknown structure type:" (symbol->string type) "is not defined in the global environment.") " "))))))))))

(define (vaquero-read-list port)
   ; lparen already swallowed
   (let loop ((token (peek-char port)) (acc '()))
      (cond
       ((eof-object? token)
          (vaquero-error 'read-error 'EOF "read error: unexpected EOF in unterminated list!"))
       ((char-whitespace? token)
          (read-char port)
          (loop (peek-char port) acc))
{'fraction_non_alphanumeric': 0.15953638997784217, 'fraction_numerical': 0.0001704448610874382, 'mean_word_length': 2.0803149606299214, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '317513', 'n_tokens_mistral': 1931, 'n_tokens_neox': 1902, 'n_words': 495}