<commit_before>;; Copyright (c) 2012-2015 <NAME>.  All rights reserved.
;; BSD 3-Clause License: http://opensource.org/licenses/BSD-3-Clause

(define (unbound-symbol? maybe-sym scope)
  (and (symbol? maybe-sym)
       (not (memq maybe-sym scope))))

(define *gensym-counter* 0)

(define (gensym prefix)
  (set! *gensym-counter* (+ 1 *gensym-counter*))
  (string->symbol (string-append prefix (number->string *gensym-counter*))))

(define (compile-expression expr scope)
  (define (unbound? maybe-sym)
    (unbound-symbol? maybe-sym scope))

  (match expr
    ((? null?) '(scm.quote ()))
    ('true #t)
    ('false #f)
    ('|{| '(scm.quote |{|))
    ('|}| '(scm.quote |}|))
    ('|;| '(scm.quote |;|))
    ((? unbound? sym) `(scm.quote ,sym))
    (('let var value body) (emit-let var value body (cons var scope)))
    (('cond clauses ...) (emit-cond clauses scope))
    ;; Remove intermediary wrapper lambdas
    (('lambda var ((? unbound? op) var)) op)
    (('lambda var body)
     `(lambda ,var ,(compile-expression body (cons var scope))))<commit_msg>Disable lambda unwrapping for now, breaks some code<commit_after>;; Copyright (c) 2012-2015 <NAME>.  All rights reserved.
;; BSD 3-Clause License: http://opensource.org/licenses/BSD-3-Clause

(define (unbound-symbol? maybe-sym scope)
  (and (symbol? maybe-sym)
       (not (memq maybe-sym scope))))

(define *gensym-counter* 0)

(define (gensym prefix)
  (set! *gensym-counter* (+ 1 *gensym-counter*))
  (string->symbol (string-append prefix (number->string *gensym-counter*))))

(define (compile-expression expr scope)
  (define (unbound? maybe-sym)
    (unbound-symbol? maybe-sym scope))

  (match expr
    ((? null?) '(scm.quote ()))
    ('true #t)
    ('false #f)
    ('|{| '(scm.quote |{|))
    ('|}| '(scm.quote |}|))
    ('|;| '(scm.quote |;|))
    ((? unbound? sym) `(scm.quote ,sym))
    (('let var value body) (emit-let var value body (cons var scope)))
    (('cond clauses ...) (emit-cond clauses scope))
    ;; Remove intermediary wrapper lambdas
    ;; FIXME: disabled for now, op may not be defined yet
    ;; (('lambda var ((? unbound? op) var)) op)
    (('lambda var body)
     `(lambda ,var ,(compile-expression body (cons var scope))))
{'fraction_non_alphanumeric': 0.19528772088808338, 'fraction_numerical': 0.010874490258269144, 'mean_word_length': 3.5809128630705396, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1500757', 'n_tokens_mistral': 799, 'n_tokens_neox': 757, 'n_words': 214}