<commit_before>#lang racket/base
(provide
  racket-syntax
  racket-quasisyntax

  syntax
  quasisyntax

  syntax?
  syntax->list
  syntax->datum
  datum->syntax

  identifier?
  ;free-identifier=?
  ;bound-identifier=?

  syntax-pair?
  syntax-vector?
  syntax-null?
  syntax-boolean?
  syntax-number?
  syntax-string?
  syntax-char?

  syntax-car
  syntax-cdr
  syntax-vector-ref

  syntax-new
  syntax-datum
  syntax-hygiene
  syntax-metadata

  ;syntax-flipmark
  ;syntax-rename
  )

(module
  utilities racket/base
  (provide (all-defined-out))

  (require
    (for-syntax racket/base)
    racket/vector
    "record.rkt"
    )

  (define-record
    syntax-new syntax? syntax-datum syntax-hygiene syntax-metadata)

  (define (racket-syntax-metadata stx)
    (vector 'racket-source-info
            (syntax-source-module stx)
            (syntax-source stx)
            (syntax-line stx)
            (syntax-column stx)
            (syntax-span stx)
            (syntax-position stx)))

  (define (syntax-new/racket datum stx)
    (syntax-new datum '() (racket-syntax-metadata stx)))

  (define (syntax-type? type? stx)
    (or (type? stx)
        (and (syntax? stx) (syntax-type? type? (syntax-datum stx)))))
  (define (syntax-pair? stx) (syntax-type? pair? stx))
  (define (syntax-vector? stx) (syntax-type? vector? stx))
  (define (syntax-null? stx) (syntax-type? null? stx))
  (define (syntax-boolean? stx) (syntax-type? boolean? stx))
  (define (syntax-number? stx) (syntax-type? number? stx))
  (define (syntax-string? stx) (syntax-type? string? stx))
  (define (syntax-char? stx) (syntax-type? char? stx))

  (define (syntax-pair-access access stx)
    (cond ((pair? stx) (access stx))
          ((syntax? stx) (syntax-pair-access access (syntax-datum stx)))
          (else (error "datum is not a syntax pair:" stx))))
  (define (syntax-car stx) (syntax-pair-access car stx))
  (define (syntax-cdr stx) (syntax-pair-access cdr stx))

  (define (syntax-vector-ref stx idx)
    (cond ((vector? stx) (vector-ref stx idx))
          ((syntax? stx) (syntax-vector-ref (syntax-datum stx) idx))
          (else (error "datum is not a syntax vector:" stx))))

  (define (syntax->list stx)
    (cond ((syntax-null? stx) '())
          ((syntax-pair? stx)
           (cons (syntax-car stx) (syntax->list (syntax-cdr stx))))
          (else (error "datum is not a syntax list:" stx))))

  (define (syntax->datum stx)
    (cond ((syntax? stx) (syntax->datum (syntax-datum stx)))
          ((pair? stx) (cons (syntax->datum (car stx))
                             (syntax->datum (cdr stx))))
          ((vector? stx) (vector-map syntax->datum stx))
          (else stx)))

  (define (datum->syntax stx datum)
    (syntax-new datum (syntax-hygiene stx) #f))

  (define (identifier? stx) (syntax-type? symbol? stx))

  (define-syntax racket-syntax (syntax-rules () ((_ s) #'s)))
  (define-syntax racket-quasisyntax (syntax-rules () ((_ s) #`s)))<commit_msg>Define hygiene marks and renaming<commit_after>#lang racket/base
(provide
  racket-syntax
  racket-quasisyntax

  syntax
  quasisyntax

  syntax?
  syntax->list
  syntax->datum
  datum->syntax

  identifier?
  ;free-identifier=?
  ;bound-identifier=?

  syntax-pair?
  syntax-vector?
  syntax-null?
  syntax-boolean?
  syntax-number?
  syntax-string?
  syntax-char?

  syntax-car
  syntax-cdr
  syntax-vector-ref

  syntax-new
  syntax-datum
  syntax-hygiene
  syntax-metadata

  syntax-mark
  syntax-rename

  mark-fresh
  renaming-fresh
  )

(module
  utilities racket/base
  (provide (all-defined-out))

  (require
    (for-syntax racket/base)
    racket/vector
    "record.rkt"
    )

  (define mark-fresh
    (let ((mark 0)) (lambda () (set! mark (+ 1 mark)) mark)))
  (define (mark? hdatum) (number? hdatum))

  (define label-fresh
    (let ((label 0)) (lambda () (set! label (+ 1 label)) label)))
  (define (renaming-fresh symbol marks) (vector symbol marks (label-fresh)))
  (define (renaming? hd) (vector? hd))
  (define (renaming-symbol r) (vector-ref r 0))
  (define (renaming-marks r) (vector-ref r 1))
  (define (renaming-label r) (vector-ref r 2))

  (define (hygiene-cons h h*)
    (cond ((null? h*) (list h))
          ((and (mark? h) (eqv? h (car h*))) (cdr h*))
          (else (cons h h*))))
  (define (hygiene-append h1 h2)
    (if (null? h1) h2
      (hygiene-cons (car h1) (hygiene-append (cdr h1) h2))))

  (define-record
    syntax-new syntax?
    syntax-datum (syntax-hygiene syntax-hygiene-set) syntax-metadata)

  (define (racket-syntax-metadata stx)
    (vector 'racket-source-info
            (syntax-source-module stx)
            (syntax-source stx)
            (syntax-line stx)
            (syntax-column stx)
            (syntax-span stx)
            (syntax-position stx)))

  (define (syntax-new/racket datum stx)
    (syntax-new datum '() (racket-syntax-metadata stx)))

  (define (syntax-type? type? stx)
    (or (type? stx)
        (and (syntax? stx) (syntax-type? type? (syntax-datum stx)))))
  (define (syntax-pair? stx) (syntax-type? pair? stx))
  (define (syntax-vector? stx) (syntax-type? vector? stx))
  (define (syntax-null? stx) (syntax-type? null? stx))
  (define (syntax-boolean? stx) (syntax-type? boolean? stx))
  (define (syntax-number? stx) (syntax-type? number? stx))
  (define (syntax-string? stx) (syntax-type? string? stx))
  (define (syntax-char? stx) (syntax-type? char? stx))

  (define (syntax-pair-access access stx)
    (cond ((pair? stx) (access stx))
          ((syntax? stx) (syntax-pair-access access (syntax-datum stx)))
          (else (error "datum is not a syntax pair:" stx))))
  (define (syntax-car stx) (syntax-pair-access car stx))
  (define (syntax-cdr stx) (syntax-pair-access cdr stx))

  (define (syntax-vector-ref stx idx)
    (cond ((vector? stx) (vector-ref stx idx))
          ((syntax? stx) (syntax-vector-ref (syntax-datum stx) idx))
          (else (error "datum is not a syntax vector:" stx))))

  (define (syntax->list stx)
    (cond ((syntax-null? stx) '())
          ((syntax-pair? stx)
           (cons (syntax-car stx) (syntax->list (syntax-cdr stx))))
          (else (error "datum is not a syntax list:" stx))))

  (define (syntax->datum stx)
    (cond ((syntax? stx) (syntax->datum (syntax-datum stx)))
          ((pair? stx) (cons (syntax->datum (car stx))
                             (syntax->datum (cdr stx))))
          ((vector? stx) (vector-map syntax->datum stx))
          (else stx)))

  (define (datum->syntax stx datum)
    (syntax-hygiene-append datum (syntax-hygiene stx)))

  (define (syntax-hygiene-append datum hygiene)
    (if (syntax? datum)
      (syntax-hygiene-set
        datum (hygiene-append hygiene (syntax-hygiene datum)))
      (syntax-new datum hygiene #f)))

  (define (identifier? stx) (syntax-type? symbol? stx))

  (define (syntax-hygiene-cons stx h)
    (syntax-hygiene-set stx (hygiene-cons h (syntax-hygiene stx))))

  (define (syntax-mark stx mark) (syntax-hygiene-cons stx mark))
  (define (syntax-rename stx renaming) (syntax-hygiene-cons stx renaming))

  (define-syntax racket-syntax (syntax-rules () ((_ s) #'s)))
  (define-syntax racket-quasisyntax (syntax-rules () ((_ s) #`s)))
{'fraction_non_alphanumeric': 0.1567861626447203, 'fraction_numerical': 0.0019528525596317478, 'mean_word_length': 2.9266155531215774, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '719657', 'n_tokens_mistral': 2775, 'n_tokens_neox': 2530, 'n_words': 663}