<commit_before>(provide ast->racket-datum)

(define (~map f ~xs)
  (cond ((pair? ~xs) (cons (f (car ~xs)) (~map f (cdr ~xs))))
        ((null? ~xs) '())
        (#t (f ~xs))))

(define (ast->racket-datum ast)
  (define alen          (- (vector-length ast) 1))
  (define (@ i)         (vector-ref ast i))
  (define (@^ i)        (ast->racket-datum (@ i)))
  (define (? tag)       (equal? tag (@ 0)))
  (define (sym s)       (vector 'symbol s))
  (define (vsym p)      (sym (string-append 'v. p)))
  (define (binding n a) (list (vsym n) (ast->racket-datum a)))
  (define (body e)      (if (and (pair? e) (equal? '#(symbol begin) (car e)))
                          (cdr e) (list e)))
  ;; Assume: set! param is a name; lambda param and apply arg are lists.
  ;; TODO: simplify quote when possible.
  (cond ((? 'quote ) (list (sym 'quote) (@ 1)))
        ((? 'var   ) (vsym (@ 1)))
        ((? 'set!  ) (list (sym 'set!) (vsym (@ 1)) (@^ 2)))
        ((? 'if    ) (list (sym 'if) (@^ 1) (@^ 2) (@^ 3)))
        ((? 'apply ) (cons (@^ 1) (map ast->racket-datum (@ 2))))
        ((? 'lambda) (list* (sym 'lambda) (~map vsym (@ 1)) (body (@^ 2))))
        ((? 'prim  ) (cons (@ 1) (map ast->racket-datum (@ 2))))
        ((? 'begin ) (cons (sym 'begin) (append (map ast->racket-datum (@ 1))
                                                (ast->racket-datum (@ 2)))))
        ((or (? 'let) (? 'letrec))
         (list* (sym (@ 0)) (map binding (@ 1) (@ 2)) (body (@^ 3))))
        (#t (error '"unknown ast:" ast))))<commit_msg>Define some racket compiler utilities<commit_after>(provide ast->racket-datum
         rkt:compile rkt:define rkt:provide rkt:require rkt:module/main)

(require ast-elaborate)

(define (~map f ~xs)
  (cond ((pair? ~xs) (cons (f (car ~xs)) (~map f (cdr ~xs))))
        ((null? ~xs) '())
        (#t (f ~xs))))
(define (sym s)  (vector 'symbol s))
(define (vsym p) (sym (string-append 'v. p)))

(define (ast->racket-datum ast)
  (define alen          (- (vector-length ast) 1))
  (define (@ i)         (vector-ref ast i))
  (define (@^ i)        (ast->racket-datum (@ i)))
  (define (? tag)       (equal? tag (@ 0)))
  (define (binding n a) (list (vsym n) (ast->racket-datum a)))
  (define (body e)      (if (and (pair? e) (equal? '#(symbol begin) (car e)))
                          (cdr e) (list e)))
  ;; Assume: set! param is a name; lambda param and apply arg are lists.
  ;; TODO: simplify quote when possible.
  (cond ((? 'quote ) (list (sym 'quote) (@ 1)))
        ((? 'var   ) (vsym (@ 1)))
        ((? 'set!  ) (list (sym 'set!) (vsym (@ 1)) (@^ 2)))
        ((? 'if    ) (list (sym 'if) (@^ 1) (@^ 2) (@^ 3)))
        ((? 'apply ) (cons (@^ 1) (map ast->racket-datum (@ 2))))
        ((? 'lambda) (list* (sym 'lambda) (~map vsym (@ 1)) (body (@^ 2))))
        ((? 'prim  ) (cons (@ 1) (map ast->racket-datum (@ 2))))
        ((? 'begin ) (cons (sym 'begin) (append (map ast->racket-datum (@ 1))
                                                (ast->racket-datum (@ 2)))))
        ((or (? 'let) (? 'letrec))
         (list* (sym (@ 0)) (map binding (@ 1) (@ 2)) (body (@^ 3))))
        (#t (error '"unknown ast:" ast))))

(define (rkt:compile ast)      (ast->racket-datum (ast-elaborate ast)))
(define (rkt:define spec rkt*) (list* (sym 'define) (~map sym spec) rkt*))
(define (rkt:provide names)    (cons (sym 'provide) (map sym names)))
(define (rkt:require names)    (cons (sym 'require) names))
(define (rkt:module/main reqs rkt*)
  (list (sym 'module) (sym '_) (sym 'racket/base)
        (rkt:provide '(main)) (rkt:require (cons 'prim.rkt reqs))
        (rkt:define '(main) rkt*)
        (list (sym 'module+) (sym 'main) (list (sym 'main)))))
{'fraction_non_alphanumeric': 0.23204570184983678, 'fraction_numerical': 0.011425462459194777, 'mean_word_length': 2.370302474793767, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '100176', 'n_tokens_mistral': 1395, 'n_tokens_neox': 1332, 'n_words': 412}