<commit_before>(module for '#%kernel

  (#%require "more-scheme.rkt"
             "misc.rkt"
             "define.rkt"
             "letstx-scheme.rkt"
             "reverse.rkt"
             '#%unsafe
             (for-syntax '#%kernel
                         "stx.rkt"
                         "qqstx.rkt"
                         "define.rkt"
                         "small-scheme.rkt"
                         "stxcase-scheme.rkt"))

  (#%provide for/fold for*/fold
             for for*
             for/list for*/list
             for/vector for*/vector
             for/lists for*/lists
             for/and for*/and
             for/or for*/or
             for/first for*/first
             for/last for*/last
             for/sum for*/sum
             for/product for*/product
             for/hash for*/hash
             for/hasheq for*/hasheq
             for/hasheqv for*/hasheqv

             for/fold/derived for*/fold/derived
             (for-syntax split-for-body)

             (rename *in-range in-range)
             (rename *in-naturals in-naturals)
             (rename *in-list in-list)
             (rename *in-mlist in-mlist)
             (rename *in-vector in-vector)
             (rename *in-string in-string)
             (rename *in-bytes in-bytes)
             (rename *in-stream in-stream)
             (rename *in-input-port-bytes in-input-port-bytes)
             (rename *in-input-port-chars in-input-port-chars)
             (rename *in-port in-port)
             (rename *in-lines in-lines)
             (rename *in-bytes-lines in-bytes-lines)
             in-hash
             in-hash-keys
             in-hash-values
             in-hash-pairs
             in-directory

             in-sequences
             in-cycle
             in-parallel
             in-values-sequence
             in-values*-sequence
             stop-before
             stop-after
             (rename *in-producer in-producer)
             (rename *in-indexed in-indexed)
             (rename *in-value in-value)

             stream?
             stream-empty?
             stream-first
             stream-rest
             prop:stream
             stream-ref ; only provided for racket/stream
             sequence->stream
             empty-stream make-do-stream

             sequence?
             sequence-generate
             sequence-generate*
             prop:sequence

             define-sequence-syntax
             make-do-sequence
             :do-in

             define-in-vector-like
             define-:vector-like-gen
             (for-syntax make-in-vector-like
                         for-clause-syntax-protect))

  ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  ;; sequence transformers:

  (begin-for-syntax
    (define-values (struct:sequence-transformer
                    make-sequence-transformer
                    sequence-transformer?
                    sequence-transformer-ref
                    sequence-transformer-set!)
      (make-struct-type 'sequence-transformer #f
                        2 0 #f
                        null (current-inspector)
                        0))

    (define (create-sequence-transformer proc1 proc2)
      (unless (and (procedure? proc1)
                   (or (procedure-arity-includes? proc1 1)
                       (procedure-arity-includes? proc1 0)))
        (raise-argument-error 'define-sequence-syntax
                              "(or/c (procedure-arity-includes/c 0) (procedure-arity-includes/c 1))"
                              0
                              proc1 proc2))
      (unless (and (procedure? proc2)
                   (procedure-arity-includes? proc2 1))
        (raise-argument-error 'define-sequence-syntax
                          "(procedure-arity-includes/c 1)"
                          1
                          proc1 proc2))
      (make-sequence-transformer
       (if (procedure-arity-includes? proc1 0)
           (lambda (stx)
             (if (identifier? stx)
                 (proc1)
                 (datum->syntax stx
                                #`(#,(proc1) . #,(cdr (syntax-e stx)))
                                stx
                                stx)))
           proc1)
       proc2))

    (define (arm-for-clause clause cert)<commit_msg>preserve lexical context, src, etc for sequence syntax expansion<commit_after>(module for '#%kernel

  (#%require "more-scheme.rkt"
             "misc.rkt"
             "define.rkt"
             "letstx-scheme.rkt"
             "reverse.rkt"
             '#%unsafe
             (for-syntax '#%kernel
                         "stx.rkt"
                         "qqstx.rkt"
                         "define.rkt"
                         "small-scheme.rkt"
                         "stxcase-scheme.rkt"))

  (#%provide for/fold for*/fold
             for for*
             for/list for*/list
             for/vector for*/vector
             for/lists for*/lists
             for/and for*/and
             for/or for*/or
             for/first for*/first
             for/last for*/last
             for/sum for*/sum
             for/product for*/product
             for/hash for*/hash
             for/hasheq for*/hasheq
             for/hasheqv for*/hasheqv

             for/fold/derived for*/fold/derived
             (for-syntax split-for-body)

             (rename *in-range in-range)
             (rename *in-naturals in-naturals)
             (rename *in-list in-list)
             (rename *in-mlist in-mlist)
             (rename *in-vector in-vector)
             (rename *in-string in-string)
             (rename *in-bytes in-bytes)
             (rename *in-stream in-stream)
             (rename *in-input-port-bytes in-input-port-bytes)
             (rename *in-input-port-chars in-input-port-chars)
             (rename *in-port in-port)
             (rename *in-lines in-lines)
             (rename *in-bytes-lines in-bytes-lines)
             in-hash
             in-hash-keys
             in-hash-values
             in-hash-pairs
             in-directory

             in-sequences
             in-cycle
             in-parallel
             in-values-sequence
             in-values*-sequence
             stop-before
             stop-after
             (rename *in-producer in-producer)
             (rename *in-indexed in-indexed)
             (rename *in-value in-value)

             stream?
             stream-empty?
             stream-first
             stream-rest
             prop:stream
             stream-ref ; only provided for racket/stream
             sequence->stream
             empty-stream make-do-stream

             sequence?
             sequence-generate
             sequence-generate*
             prop:sequence

             define-sequence-syntax
             make-do-sequence
             :do-in

             define-in-vector-like
             define-:vector-like-gen
             (for-syntax make-in-vector-like
                         for-clause-syntax-protect))

  ;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
  ;; sequence transformers:

  (begin-for-syntax
    (define-values (struct:sequence-transformer
                    make-sequence-transformer
                    sequence-transformer?
                    sequence-transformer-ref
                    sequence-transformer-set!)
      (make-struct-type 'sequence-transformer #f
                        2 0 #f
                        null (current-inspector)
                        0))

    (define (create-sequence-transformer proc1 proc2)
      (unless (and (procedure? proc1)
                   (or (procedure-arity-includes? proc1 1)
                       (procedure-arity-includes? proc1 0)))
        (raise-argument-error 'define-sequence-syntax
                              "(or/c (procedure-arity-includes/c 0) (procedure-arity-includes/c 1))"
                              0
                              proc1 proc2))
      (unless (and (procedure? proc2)
                   (procedure-arity-includes? proc2 1))
        (raise-argument-error 'define-sequence-syntax
                          "(procedure-arity-includes/c 1)"
                          1
                          proc1 proc2))
      (make-sequence-transformer
       (if (procedure-arity-includes? proc1 0)
           (lambda (stx)
             (if (identifier? stx)
                 (proc1)
                 (datum->syntax stx
                                ;; Use cons, not #`(#,op #,@args), to avoid replacing implicit #%app binding
                                (cons (proc1) (cdr (syntax-e stx)))
                                stx
                                stx)))
           proc1)
       proc2))

    (define (arm-for-clause clause cert)
{'fraction_non_alphanumeric': 0.11112373593909783, 'fraction_numerical': 0.006362913305306216, 'mean_word_length': 1.1067496409765438, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5807844', 'n_tokens_mistral': 2431, 'n_tokens_neox': 2275, 'n_words': 453}