<commit_before>;; Basic PEG-ish parsing

;; TODO: fuller error reporting
;; TODO: memoize
;; TODO: delay semantic actions until final success

;; Glossary:
;;  p, q       parsing expression
;;  text       input sequence
;;  far        the rightmost index tentatively eaten up to in text
;;             (used for error reporting)
;;  i, j       index into text
;;  vals, vs   list of parsed values

(define (fail text far i vals)
  (make failure
    (.display ()
      (display "failed: ")
      (write (.slice text 0 far))
      (display "/")
      (write (.slice text far)))
    (.invert ()          empty)
    (.else (p text j vs) (p text far j vs))
    (.continue (p)       failure)
    (.capture-from (j)   failure)
    (.prefix (pre-vals)  failure)
    (.leftovers ()       (error "Parsing failed" failure))
    (.opt-results ()     #f)
    (.result ()          (error "Parsing failed" failure))))

(define (empty text far i vals)
  (make success
    (.display () 
      (write (.slice text i))
      (display " ")
      (write vals))
    (.invert ()          fail)
    (.else (p text j vs) success)
    (.continue (p)       (p text far i vals))
    (.capture-from (j)   (empty text far i `(,@vals ,(.slice text j i))))
    (.prefix (pre-vals)  (empty text far i (chain pre-vals vals)))
    (.leftovers ()       i)
    (.opt-results ()     vals)
    (.result ()
      (if (= 1 (.count vals))
          ('first vals)
          (error "Wrong # of results" vals)))))

(define ((invert p) text far i vals)
  ((.invert (p text far i vals))
   text far i vals))

(define ((capture p) text far i vals)
  (.capture-from (p text far i vals) i))

(define (folded<- combine)
  (given arguments
    (foldr1 combine arguments)))

(let either
  (folded<- (given (p q)
              (given (text far i vals)
                (.else (p text far i vals) q text i vals)))))

(let then
  (folded<- (given (p q)
              (given (text far i vals)
                (.continue (p text far i vals) q)))))
<commit_msg>use curried define<commit_after>;; Basic PEG-ish parsing

;; TODO: fuller error reporting
;; TODO: memoize
;; TODO: delay semantic actions until final success

;; Glossary:
;;  p, q       parsing expression
;;  text       input sequence
;;  far        the rightmost index tentatively eaten up to in text
;;             (used for error reporting)
;;  i, j       index into text
;;  vals, vs   list of parsed values

(define (fail text far i vals)
  (make failure
    (.display ()
      (display "failed: ")
      (write (.slice text 0 far))
      (display "/")
      (write (.slice text far)))
    (.invert ()          empty)
    (.else (p text j vs) (p text far j vs))
    (.continue (p)       failure)
    (.capture-from (j)   failure)
    (.prefix (pre-vals)  failure)
    (.leftovers ()       (error "Parsing failed" failure))
    (.opt-results ()     #f)
    (.result ()          (error "Parsing failed" failure))))

(define (empty text far i vals)
  (make success
    (.display () 
      (write (.slice text i))
      (display " ")
      (write vals))
    (.invert ()          fail)
    (.else (p text j vs) success)
    (.continue (p)       (p text far i vals))
    (.capture-from (j)   (empty text far i `(,@vals ,(.slice text j i))))
    (.prefix (pre-vals)  (empty text far i (chain pre-vals vals)))
    (.leftovers ()       i)
    (.opt-results ()     vals)
    (.result ()
      (if (= 1 (.count vals))
          ('first vals)
          (error "Wrong # of results" vals)))))

(define ((invert p) text far i vals)
  ((.invert (p text far i vals))
   text far i vals))

(define ((capture p) text far i vals)
  (.capture-from (p text far i vals) i))

(define (folded<- combine)
  (given arguments
    (foldr1 combine arguments)))

(let either
  (folded<- (define ((either p q) text far i vals)
              (.else (p text far i vals) q text i vals))))

(let then
  (folded<- (define ((then p q) text far i vals)
              (.continue (p text far i vals) q))))

{'fraction_non_alphanumeric': 0.145985401459854, 'fraction_numerical': 0.0015101938082053865, 'mean_word_length': 2.102263856362217, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5164737', 'n_tokens_mistral': 1359, 'n_tokens_neox': 1265, 'n_words': 500}