<commit_before>#lang racket/base
(provide (all-from-out 'interop) (all-from-out 'common)
         interop-eval racket-eval)

(module interop racket/base
  (provide
    local-path library-path s->ns ns->s read/file write/file racket-datum
    lift lower lower-arg0 $apply
    mvector? make-mvector mvector=? mvector-length mvector-ref mvector-set!
    mvector->vector string->vector vector->string procedure=? number=?
    alist-get alist-remove* alist-at list-at
    nscm-quote nscm-quasiquote
    (rename-out (equal? nscm-equal?) (member nscm-member) (assoc nscm-assoc)))

  (require racket/path racket/port racket/vector (for-syntax racket/base))

  (define (local-path rpath)
    (build-path (path-only (path->complete-path (find-system-path 'run-file)))
                rpath))
  (define (library-path library-name module-name)
    (local-path
      (build-path "lib" (symbol->string library-name)
                  (string-append (symbol->string module-name) ".scm"))))

  (define (s->ns d)
    (cond ((symbol? d) (symbol->string d))
          ((pair? d) (cons (s->ns (car d)) (s->ns (cdr d))))
          ((vector? d) (vector-map s->ns d))
          (else d)))

  (define (ns->s d)
    (cond ((string? d) (string->symbol d))
          ((pair? d) (cons (ns->s (car d)) (ns->s (cdr d))))
          ((vector? d) (vector-map ns->s d))
          (else d)))

  (define (ns->s/write d)
    (cond ((string? d) (string->symbol d))
          ((pair? d) (cons (ns->s (car d)) (ns->s (cdr d))))
          ((vector? d) (vector-map ns->s d))
          ((or (boolean? d) (null? d) (number? d)) d)
          (else (error "cannot write:" d))))

  (define (read/file path)
    (call-with-input-file
      path (lambda (in) (let loop ((rbody '()))
                          (define datum (ns->s (read in)))
                          (if (eof-object? datum) (reverse rbody)
                            (loop (cons datum rbody)))))))
  (define (write/file path d)
    (call-with-output-file path (lambda (out) (write (ns->s/write d) out))))

  (define (racket-datum form)
    (define (? tag) (eq? (vector-ref form 0) tag))
    (cond ((pair? form) (cons (racket-datum (car form))
                              (racket-datum (cdr form))))
          ((not (vector? form)) form)
          ((? 'quote)   (vector-ref form 1))
          ((? 'vector)  (vector-map racket-datum (vector-ref form 1)))
          ((? 'symbol)  (string->symbol (vector-ref form 1)))
          ((? 'keyword) (string->keyword (vector-ref form 1)))
          ((? 'char)    (call-with-input-string
                          (string-append "#\\" (vector-ref form 1))
                          (lambda (in) (read in))))
          (#t (error "invalid racket-datum form:" form))))

  ;; For safe interop, provided definitions must not override Racket names.
  (define (equal? a b)
    (or (eqv? a b)
        (and (string? a) (string? b) (string=? a b))
        (and (pair? a) (pair? b)
             (equal? (car a) (car b))
             (equal? (cdr a) (cdr b)))
        (and (vector? a) (vector? b)
             (equal? (vector->list a) (vector->list b)))))
  (define (member v xs) (memf (lambda (x) (equal? x v)) xs))
  (define (assoc k xs) (cond ((null? xs) #f)
                             ((equal? k (caar xs)) (car xs))
                             (else (assoc k (cdr xs)))))
<commit_msg>Factor out read/port, write/port; fix racket-datum<commit_after>#lang racket/base
(provide (all-from-out 'interop) (all-from-out 'common)
         interop-eval racket-eval)

(module interop racket/base
  (provide
    local-path library-path s->ns ns->s
    read/port read/file write/port write/file racket-datum
    lift lower lower-arg0 $apply
    mvector? make-mvector mvector=? mvector-length mvector-ref mvector-set!
    mvector->vector string->vector vector->string procedure=? number=?
    alist-get alist-remove* alist-at list-at
    nscm-quote nscm-quasiquote
    (rename-out (equal? nscm-equal?) (member nscm-member) (assoc nscm-assoc)))

  (require racket/path racket/port racket/vector (for-syntax racket/base))

  (define (local-path rpath)
    (build-path (path-only (path->complete-path (find-system-path 'run-file)))
                rpath))
  (define (library-path library-name module-name)
    (local-path
      (build-path "lib" (symbol->string library-name)
                  (string-append (symbol->string module-name) ".scm"))))

  (define (s->ns d)
    (cond ((symbol? d) (symbol->string d))
          ((pair? d) (cons (s->ns (car d)) (s->ns (cdr d))))
          ((vector? d) (vector-map s->ns d))
          ((or (char? d) (keyword? d)) (error "invalid nscheme datum:" d))
          (else d)))

  (define (ns->s d)
    (cond ((string? d) (string->symbol d))
          ((pair? d) (cons (ns->s (car d)) (ns->s (cdr d))))
          ((vector? d) (vector-map ns->s d))
          (else d)))

  (define (ns->s/write d)
    (cond ((string? d) (string->symbol d))
          ((pair? d) (cons (ns->s (car d)) (ns->s (cdr d))))
          ((vector? d) (vector-map ns->s d))
          ((or (boolean? d) (null? d) (number? d)) d)
          (else (error "cannot write:" d))))

  (define (read/port in)
    (let loop ((rbody '()))
      (define datum (ns->s (read in)))
      (if (eof-object? datum) (reverse rbody) (loop (cons datum rbody)))))
  (define (read/file path)    (call-with-input-file path read/port))
  (define (write/port out d)  (write (ns->s/write d) out))
  (define (write/file path d) (call-with-output-file path write/port))

  (define (racket-datum form)
    (define (? tag) (equal? (vector-ref form 0) tag))
    (cond ((pair? form) (cons (racket-datum (car form))
                              (racket-datum (cdr form))))
          ((not (vector? form)) form)
          ((? "quote")   (vector-ref form 1))
          ((? "vector")  (vector-map racket-datum (vector-ref form 1)))
          ((? "symbol")  (string->symbol (vector-ref form 1)))
          ((? "keyword") (string->keyword (vector-ref form 1)))
          ((? "char")    (call-with-input-string
                           (string-append "#\\" (vector-ref form 1))
                           (lambda (in) (read in))))
          (#t (error "invalid racket-datum form:" form))))

  ;; For safe interop, provided definitions must not override Racket names.
  (define (equal? a b)
    (or (eqv? a b)
        (and (string? a) (string? b) (string=? a b))
        (and (pair? a) (pair? b)
             (equal? (car a) (car b))
             (equal? (cdr a) (cdr b)))
        (and (vector? a) (vector? b)
             (equal? (vector->list a) (vector->list b)))))
  (define (member v xs) (memf (lambda (x) (equal? x v)) xs))
  (define (assoc k xs) (cond ((null? xs) #f)
                             ((equal? k (caar xs)) (car xs))
                             (else (assoc k (cdr xs)))))

{'fraction_non_alphanumeric': 0.17977362928119947, 'fraction_numerical': 0.002057915625459356, 'mean_word_length': 2.1943661971830988, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2714879', 'n_tokens_mistral': 2373, 'n_tokens_neox': 2295, 'n_words': 696}