<commit_before>#lang racket/base
(provide
  std0
  std0-module
  symbol-table-string
  unsafe1-parse
  unsafe1-module
  )

(require
  "denotation.rkt"
  "linking.rkt"
  "operation.rkt"
  "parsing.rkt"
  "term.rkt"
  "unsafe0.rkt"
  gregr-misc/dict
  gregr-misc/maybe
  gregr-misc/record
  gregr-misc/sugar
  racket/function
  racket/match
  racket/string
  )

(module+ test
  (require
    rackunit
    ))

(define pcons '(lambda (hd tl) (pair hd tl)))

(define tag:symbol  '(pair 0 (pair 0 (pair 0 ()))))
(define tag:boolean '(pair 0 (pair 0 (pair 1 ()))))
(define tag:nil     '(pair 0 (pair 1 (pair 0 ()))))
(define tag:cons    '(pair 0 (pair 1 (pair 1 ()))))
(define tag:integer '(pair 1 (pair 0 (pair 0 ()))))

(define bits-nil '(pair 0 ()))
(define bits '(lambda (b0 bs) (pair 1 (pair b0 bs))))

(define (nat->bits n (invert? #f))
  (if (= 0 n) bits-nil
    `(,bits ,(if ((if invert? not identity) (= 0 (modulo n 2))) 0 1)
            ,(nat->bits (quotient n 2) invert?))))

(define (nat->symbol n)
  (step-complete (unsafe0-parse `(,pcons ,tag:symbol ,(nat->bits n)))))

(define (int->integer i)
  (step-complete (unsafe0-parse
    `(,pcons ,tag:integer
       (,pcons ,(if (< i 0) 1 0)
         ,(nat->bits (if (< i 0) (- (+ i 1)) i) (< i 0)))))))

(define std0-module (unsafe0-module
  `((identity (lambda (x) x))
    (const    (lambda (k _) k))
    (compose  (lambda (f g x) (f (g x))))
    (fix (lambda (f) ((lambda (d) (d d))
                      (lambda (x) (f (lambda (a) (x x a)))))))

    (pcons    ,pcons)
    (phead    (lambda (pr)    (unpair 0 pr)))
    (ptail    (lambda (pr)    (unpair 1 pr)))
    (psecond  (lambda (pr)    (phead (ptail pr))))
    (pthird   (lambda (pr)    (phead (ptail (ptail pr)))))
<commit_msg>Avoid annotating low-level literal constructions<commit_after>#lang racket/base
(provide
  std0
  std0-module
  symbol-table-string
  unsafe1-parse
  unsafe1-module
  )

(require
  "denotation.rkt"
  "linking.rkt"
  "operation.rkt"
  "parsing.rkt"
  "term.rkt"
  "unsafe0.rkt"
  gregr-misc/dict
  gregr-misc/maybe
  gregr-misc/record
  gregr-misc/sugar
  racket/function
  racket/match
  racket/string
  )

(module+ test
  (require
    rackunit
    ))

(define pcons '(lambda (hd tl) (pair hd tl)))

(define tag:symbol  '(pair 0 (pair 0 (pair 0 ()))))
(define tag:boolean '(pair 0 (pair 0 (pair 1 ()))))
(define tag:nil     '(pair 0 (pair 1 (pair 0 ()))))
(define tag:cons    '(pair 0 (pair 1 (pair 1 ()))))
(define tag:integer '(pair 1 (pair 0 (pair 0 ()))))

(define bits-nil '(pair 0 ()))
(define bits '(lambda (b0 bs) (pair 1 (pair b0 bs))))

(define (nat->bits n (invert? #f))
  (if (= 0 n) bits-nil
    `(,bits ,(if ((if invert? not identity) (= 0 (modulo n 2))) 0 1)
            ,(nat->bits (quotient n 2) invert?))))

(define (nat->symbol n)
  (strip-annotations
    (step-complete (unsafe0-parse `(,pcons ,tag:symbol ,(nat->bits n))))))

(define (int->integer i)
  (strip-annotations (step-complete (unsafe0-parse
    `(,pcons ,tag:integer
       (,pcons ,(if (< i 0) 1 0)
         ,(nat->bits (if (< i 0) (- (+ i 1)) i) (< i 0))))))))

(define std0-module (unsafe0-module
  `((identity (lambda (x) x))
    (const    (lambda (k _) k))
    (compose  (lambda (f g x) (f (g x))))
    (fix (lambda (f) ((lambda (d) (d d))
                      (lambda (x) (f (lambda (a) (x x a)))))))

    (pcons    ,pcons)
    (phead    (lambda (pr)    (unpair 0 pr)))
    (ptail    (lambda (pr)    (unpair 1 pr)))
    (psecond  (lambda (pr)    (phead (ptail pr))))
    (pthird   (lambda (pr)    (phead (ptail (ptail pr)))))

{'fraction_non_alphanumeric': 0.19402985074626866, 'fraction_numerical': 0.023655308363841172, 'mean_word_length': 2.844155844155844, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1418902', 'n_tokens_mistral': 1519, 'n_tokens_neox': 1400, 'n_words': 418}