<commit_before>; general assertions about reconstruction:
; a varref can only occur at the top of a mark-list
; a varref at the top of the mark-list must either be a top-level-variable
;  or have a value in some mark somewhere (or both).

(module reconstruct scheme/base
  (require (prefix-in kernel: syntax/kerncase)
           mzlib/list
           mzlib/etc
	   mzlib/contract
           scheme/match
           "marks.ss"
           "model-settings.ss"
           "shared.ss"
           "my-macros.ss"
           (for-syntax scheme/base)
           #;(file "/Users/clements/clements/scheme-scraps/eli-debug.ss"))

  (provide/contract 
   [reconstruct-completed (syntax? 
                           (or/c (listof natural-number/c) false/c)
                           (-> (listof any/c))
                           render-settings? 
                           . -> .
                           (vector/c syntax? boolean?))]
   [hide-completed? (syntax? . -> . boolean?)]
   
   ;; front ends for reconstruct-current
   [reconstruct-left-side (mark-list?
                           (or/c (listof any/c) false/c)
                           render-settings?
                           . -> .
                           syntax?)]
   [reconstruct-right-side (mark-list?
                            (or/c (listof any/c) false/c)
                            render-settings?
                            . -> .
                            syntax?)]
   [reconstruct-double-break (mark-list?
                              render-settings?
                              . -> . 
                              (list/c syntax? syntax?))]
   
   [final-mark-list? (-> mark-list? boolean?)]
   [skip-step? (-> break-kind? (or/c mark-list? false/c) render-settings? boolean?)]
   [step-was-app? (-> mark-list? boolean?)]
   
   [reset-special-values (-> any)])
  
  (define nothing-so-far (gensym "nothing-so-far-"))

  ; the let-glump is a structure that contains the reconstruct-time data about
  ; a let-binding; that is, the names on the left-hand-side, the expression on
  ; the right-hand side, and the values computed.
  
  (define-struct let-glump (name-set exp val-set))

  ; split-list : ('a -> boolean) (listof 'a) -> (2vals (listof 'a) (listof 'a))
  ; split-list splits a list into two lists at the first element s.t. (fn element) => true).
  ; that is, split-list yields the lists A and B such that (append A B) gives the original
  ; list, and (fn element) => false for all elements in A, and B is either empty or
  ; (fn (car B)) => true
  
 (define (split-list fn lst)
    (let loop ([remaining lst] [so-far null]) 
      (cond [(null? remaining)
             (2vals (reverse so-far) null)]
            [else
             (if (fn (car remaining))
                 (2vals (reverse so-far) remaining)
                 (loop (cdr remaining) (cons (car remaining) so-far)))])))
  
  ; test cases
  ; (test (2vals '(93 4 2) '(0 2 1)) split-list (lambda (x) (= x 0)) '(93 4 2 0 2 1))
  ; (test (2vals '(3 4 5) '()) split-list (lambda (x) (= x 0)) '(3 4 5))
        
  ; n-split-list : num ('a list) -> ('a list) ('a list)
  ; n-split-list splits a given list A into two lists B and C, such that B contains the
  ; first n elements of A, and C contains the rest.

  (define (n-split-list num lst)
    (when (> num (length lst))
      (error 'n-split-list "can't split list ~a after ~ath element; not long enough" lst num))
    (let loop ([count num] [remaining lst] [so-far null])
      (if (= count 0)
          (2vals (reverse so-far) remaining)
          (loop (- count 1) (cdr remaining) (cons (car remaining) so-far)))))
  
  ; test cases
  ; (test (2vals '(a b c) '(d e f)) n-split-list 3 '(a b c d e f))
  
  
  (define (mark-as-highlight stx)
    (stepper-syntax-property stx 'stepper-highlight #t))
                                                     ;               
                                                     ;               
 ; ;;  ;;;    ;;;   ;;;   ; ;;         ;   ;   ;;;   ;  ;   ;   ;;;  
 ;;   ;   ;  ;     ;   ;  ;;  ;        ;   ;  ;   ;  ;  ;   ;  ;   ; 
 ;    ;   ;  ;     ;   ;  ;   ;         ; ;       ;  ;  ;   ;  ;   ; 
 ;    ;;;;;  ;     ;   ;  ;   ;  ;;;;;  ; ;    ;;;;  ;  ;   ;  ;;;;; 
 ;    ;      ;     ;   ;  ;   ;         ; ;   ;   ;  ;  ;   ;  ;     
 ;    ;      ;     ;   ;  ;   ;         ;;    ;   ;  ;  ;  ;;  ;     
 ;     ;;;;   ;;;   ;;;   ;   ;          ;     ;;;;; ;   ;; ;   ;;;; 
                                                                     
                                                                     
  ; recon-value print-converts a value.  If the value is a closure, recon-value
  ; prints the name attached to the procedure, unless we're on the right-hand-side
  ; of a let, or unless there _is_ no name.
  
  (define recon-value
    (opt-lambda (val render-settings [assigned-name #f])
      (if (hash-ref finished-xml-box-table val (lambda () #f))
          (stepper-syntax-property #`(quote #,val) 'stepper-xml-value-hint 'from-xml-box)
          (let ([closure-record (closure-table-lookup val (lambda () #f))])     
            (if closure-record
                (let* ([mark (closure-record-mark closure-record)]
                       [base-name (closure-record-name closure-record)])
                  (if base-name
                      (let* ([lifted-index (closure-record-lifted-index closure-record)]
                             [name (if lifted-index
                                       (construct-lifted-name base-name lifted-index)
                                       base-name)])
                        (if (and assigned-name (free-identifier=? base-name assigned-name))
                            (recon-source-expr (mark-source mark) (list mark) null null render-settings)
                            #`#,name))
                      (recon-source-expr (mark-source mark) (list mark) null null render-settings)))
                (let* ([rendered ((render-settings-render-to-sexp render-settings) val)])
                  (if (symbol? rendered)
                      #`#,rendered
                      #`(quote #,rendered))))))))
    
  (define (final-mark-list? mark-list)
    (and (not (null? mark-list)) (eq? (mark-label (car mark-list)) 'final)))


       ;      ;                                                                     ;;;
       ;                                                          ;                    ;
  ;;;  ;   ;  ;  ; ;;;         ;    ; ;    ; ;    ;          ;;; ;;;;  ;;;   ; ;;;     ;
 ;     ;  ;   ;  ;;   ;         ;  ;   ;  ;   ;  ;          ;     ;   ;   ;  ;;   ;    ;
 ;     ; ;    ;  ;    ;          ;;     ;;     ;;           ;     ;   ;   ;  ;    ;   ;
  ;;   ;;     ;  ;    ;  ;;;;;   ;;     ;;     ;;    ;;;;;   ;;   ;   ;;;;;  ;    ;  ;
    ;  ; ;    ;  ;    ;          ;;     ;;     ;;              ;  ;   ;      ;    ;  ;
    ;  ;  ;   ;  ;;   ;         ;  ;   ;  ;   ;  ;             ;  ;   ;      ;;   ;
 ;;;   ;   ;  ;  ; ;;;         ;    ; ;    ; ;    ;         ;;;    ;;  ;;;;  ; ;;;   ;
                 ;                                                           ;
                 ;                                                           ;

  (define (skip-step? break-kind mark-list render-settings)
    (case break-kind
      [(result-value-break)
       (and (pair? mark-list)
            (let ([expr (mark-source (car mark-list))])
              (stepper-syntax-property expr 'stepper-hide-reduction)))]
      [(result-exp-break)
       ;; skip if clauses that are the result of and/or reductions
       (let ([and/or-clauses-consumed (stepper-syntax-property (mark-source (car mark-list)) 'stepper-and/or-clauses-consumed)])
         (and and/or-clauses-consumed
              (> and/or-clauses-consumed 0)))]
      [(normal-break normal-break/values)
       (skip-redex-step? mark-list render-settings)]
      [(double-break)
       (or 
        ;; don't stop for a double-break on a let that is the expansion of a 'begin'
        (let ([expr (mark-source (car mark-list))])
          (or (eq? (stepper-syntax-property expr 'stepper-hint) 'comes-from-begin)
              (stepper-syntax-property expr 'stepper-skip-double-break)))
        (not (render-settings-lifting? render-settings)))]
      [(expr-finished-break define-struct-break late-let-break) #f]))
  
  (define (skip-redex-step? mark-list render-settings)
    
    (define (varref-skip-step? varref)
      (with-handlers ([exn:fail:contract:variable? (lambda (dc-exn) #f)])
        (let ([val (lookup-binding mark-list varref)])
          (equal? (syntax-object->interned-datum (recon-value val render-settings))
                  (syntax-object->interned-datum (case (stepper-syntax-property varref 'stepper-binding-type)
                                                   ([let-bound]
                                                    (binding-lifted-name mark-list varref))
                                                   ([non-lexical]
                                                    varref)
                                                   (else
                                                    (error 'varref-skip-step? "unexpected value for stepper-binding-type: ~e for variable: ~e\n"<commit_msg>fussing with test results<commit_after>; general assertions about reconstruction:
; a varref can only occur at the top of a mark-list
; a varref at the top of the mark-list must either be a top-level-variable
;  or have a value in some mark somewhere (or both).

(module reconstruct scheme/base
  (require (prefix-in kernel: syntax/kerncase)
           mzlib/list
           mzlib/etc
	   mzlib/contract
           scheme/match
           "marks.ss"
           "model-settings.ss"
           "shared.ss"
           "my-macros.ss"
           (for-syntax scheme/base)
           #;(file "/Users/clements/clements/scheme-scraps/eli-debug.ss"))

  (provide/contract 
   [reconstruct-completed (syntax? 
                           (or/c (listof natural-number/c) false/c)
                           (-> (listof any/c))
                           render-settings? 
                           . -> .
                           (vector/c syntax? boolean?))]
   [hide-completed? (syntax? . -> . boolean?)]
   
   ;; front ends for reconstruct-current
   [reconstruct-left-side (mark-list?
                           (or/c (listof any/c) false/c)
                           render-settings?
                           . -> .
                           syntax?)]
   [reconstruct-right-side (mark-list?
                            (or/c (listof any/c) false/c)
                            render-settings?
                            . -> .
                            syntax?)]
   [reconstruct-double-break (mark-list?
                              render-settings?
                              . -> . 
                              (list/c syntax? syntax?))]
   
   [final-mark-list? (-> mark-list? boolean?)]
   [skip-step? (-> break-kind? (or/c mark-list? false/c) render-settings? boolean?)]
   [step-was-app? (-> mark-list? boolean?)]
   
   [reset-special-values (-> any)])
  
  (define nothing-so-far (gensym "nothing-so-far-"))

  ; the let-glump is a structure that contains the reconstruct-time data about
  ; a let-binding; that is, the names on the left-hand-side, the expression on
  ; the right-hand side, and the values computed.
  
  (define-struct let-glump (name-set exp val-set))

  ; split-list : ('a -> boolean) (listof 'a) -> (2vals (listof 'a) (listof 'a))
  ; split-list splits a list into two lists at the first element s.t. (fn element) => true).
  ; that is, split-list yields the lists A and B such that (append A B) gives the original
  ; list, and (fn element) => false for all elements in A, and B is either empty or
  ; (fn (car B)) => true
  
 (define (split-list fn lst)
    (let loop ([remaining lst] [so-far null]) 
      (cond [(null? remaining)
             (2vals (reverse so-far) null)]
            [else
             (if (fn (car remaining))
                 (2vals (reverse so-far) remaining)
                 (loop (cdr remaining) (cons (car remaining) so-far)))])))
  
  ; test cases
  ; (test (2vals '(93 4 2) '(0 2 1)) split-list (lambda (x) (= x 0)) '(93 4 2 0 2 1))
  ; (test (2vals '(3 4 5) '()) split-list (lambda (x) (= x 0)) '(3 4 5))
        
  ; n-split-list : num ('a list) -> ('a list) ('a list)
  ; n-split-list splits a given list A into two lists B and C, such that B contains the
  ; first n elements of A, and C contains the rest.

  (define (n-split-list num lst)
    (when (> num (length lst))
      (error 'n-split-list "can't split list ~a after ~ath element; not long enough" lst num))
    (let loop ([count num] [remaining lst] [so-far null])
      (if (= count 0)
          (2vals (reverse so-far) remaining)
          (loop (- count 1) (cdr remaining) (cons (car remaining) so-far)))))
  
  ; test cases
  ; (test (2vals '(a b c) '(d e f)) n-split-list 3 '(a b c d e f))
  
  
  (define (mark-as-highlight stx)
    (stepper-syntax-property stx 'stepper-highlight #t))
                                                     ;               
                                                     ;               
 ; ;;  ;;;    ;;;   ;;;   ; ;;         ;   ;   ;;;   ;  ;   ;   ;;;  
 ;;   ;   ;  ;     ;   ;  ;;  ;        ;   ;  ;   ;  ;  ;   ;  ;   ; 
 ;    ;   ;  ;     ;   ;  ;   ;         ; ;       ;  ;  ;   ;  ;   ; 
 ;    ;;;;;  ;     ;   ;  ;   ;  ;;;;;  ; ;    ;;;;  ;  ;   ;  ;;;;; 
 ;    ;      ;     ;   ;  ;   ;         ; ;   ;   ;  ;  ;   ;  ;     
 ;    ;      ;     ;   ;  ;   ;         ;;    ;   ;  ;  ;  ;;  ;     
 ;     ;;;;   ;;;   ;;;   ;   ;          ;     ;;;;; ;   ;; ;   ;;;; 
                                                                     
                                                                     
  ; recon-value print-converts a value.  If the value is a closure, recon-value
  ; prints the name attached to the procedure, unless we're on the right-hand-side
  ; of a let, or unless there _is_ no name.
  
  (define recon-value
    (opt-lambda (val render-settings [assigned-name #f])
      (if (hash-ref finished-xml-box-table val (lambda () #f))
          (stepper-syntax-property #`(quote #,val) 'stepper-xml-value-hint 'from-xml-box)
          (let ([closure-record (closure-table-lookup val (lambda () #f))])     
            (if closure-record
                (let* ([mark (closure-record-mark closure-record)]
                       [base-name (closure-record-name closure-record)])
                  (if base-name
                      (let* ([lifted-index (closure-record-lifted-index closure-record)]
                             [name (if lifted-index
                                       (construct-lifted-name base-name lifted-index)
                                       base-name)])
                        (if (and assigned-name (free-identifier=? base-name assigned-name))
                            (recon-source-expr (mark-source mark) (list mark) null null render-settings)
                            #`#,name))
                      (recon-source-expr (mark-source mark) (list mark) null null render-settings)))
                (let* ([rendered ((render-settings-render-to-sexp render-settings) val)])
                  (if (symbol? rendered)
                      #`#,rendered
                      #`(quote #,rendered))))))))
    
  (define (final-mark-list? mark-list)
    (and (not (null? mark-list)) (eq? (mark-label (car mark-list)) 'final)))


       ;      ;                                                                     ;;;
       ;                                                          ;                    ;
  ;;;  ;   ;  ;  ; ;;;         ;    ; ;    ; ;    ;          ;;; ;;;;  ;;;   ; ;;;     ;
 ;     ;  ;   ;  ;;   ;         ;  ;   ;  ;   ;  ;          ;     ;   ;   ;  ;;   ;    ;
 ;     ; ;    ;  ;    ;          ;;     ;;     ;;           ;     ;   ;   ;  ;    ;   ;
  ;;   ;;     ;  ;    ;  ;;;;;   ;;     ;;     ;;    ;;;;;   ;;   ;   ;;;;;  ;    ;  ;
    ;  ; ;    ;  ;    ;          ;;     ;;     ;;              ;  ;   ;      ;    ;  ;
    ;  ;  ;   ;  ;;   ;         ;  ;   ;  ;   ;  ;             ;  ;   ;      ;;   ;
 ;;;   ;   ;  ;  ; ;;;         ;    ; ;    ; ;    ;         ;;;    ;;  ;;;;  ; ;;;   ;
                 ;                                                           ;
                 ;                                                           ;

  (define (skip-step? break-kind mark-list render-settings)
    (case break-kind
      [(result-value-break)
       #f]
      [(result-exp-break)
       ;; skip if clauses that are the result of and/or reductions
       (let ([and/or-clauses-consumed (stepper-syntax-property (mark-source (car mark-list)) 'stepper-and/or-clauses-consumed)])
         (and and/or-clauses-consumed
              (> and/or-clauses-consumed 0)))]
      [(normal-break normal-break/values)
       (skip-redex-step? mark-list render-settings)]
      [(double-break)
       (or 
        ;; don't stop for a double-break on a let that is the expansion of a 'begin'
        (let ([expr (mark-source (car mark-list))])
          (or (eq? (stepper-syntax-property expr 'stepper-hint) 'comes-from-begin)
              (stepper-syntax-property expr 'stepper-skip-double-break)))
        (not (render-settings-lifting? render-settings)))]
      [(expr-finished-break define-struct-break late-let-break) #f]))
  
  ;; skip-redex-step : mark-list? render-settings? -> boolean?
  (define (skip-redex-step? mark-list render-settings)
    
    (define (varref-skip-step? varref)
      (with-handlers ([exn:fail:contract:variable? (lambda (dc-exn) #f)])
        (let ([val (lookup-binding mark-list varref)])
          (equal? (syntax-object->interned-datum (recon-value val render-settings))
                  (syntax-object->interned-datum (case (stepper-syntax-property varref 'stepper-binding-type)
                                                   ([let-bound]
                                                    (binding-lifted-name mark-list varref))
                                                   ([non-lexical]
                                                    varref)
                                                   (else
                                                    (error 'varref-skip-step? "unexpected value for stepper-binding-type: ~e for variable: ~e\n"
{'fraction_non_alphanumeric': 0.14298121810878298, 'fraction_numerical': 0.0035826728911084573, 'mean_word_length': 1.2177681473456121, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 44, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 104, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4027558', 'n_tokens_mistral': 5655, 'n_tokens_neox': 5328, 'n_words': 1278}