<commit_before>;; Generate strings that match a regex.
;; From Udacity CS212 Python code by <NAME>.
;; Glossary:
;;    r, s   regex (i.e. a thing built by the constructors below)
;;    Ns     set of nonnegative integers (i.e. lengths)

;; Return the strings matching r whose length is in Ns.
(to (generate r Ns)
  (sort-by-key (r Ns)
               (given (str) `(,str.count ,str))))

(let none   (set<-))
(let just-0 (set<- 0))

(to (literal str)
  (let just-str (set<- str))
  (given (Ns)
    (if (Ns .maps? str.count) just-str none)))

(to (star r)
  (let r1 (nonempty r))
  (begin r1* ()
    (given (Ns)
      ((maybe (then r1 (r1*))) Ns))))

(to (maybe r) (either empty r))

(to ((nonempty r) Ns) (r (Ns .difference just-0)))
(to ((either r s) Ns) ((r Ns) .union (s Ns)))

(to (one-of chars)
  (let set (call set<- (as-list chars)))
  (given (Ns)
    (if (Ns .maps? 1) set null)))

(to ((then r s) Ns)
  ;; Return the set of matches to rs whose total length is in Ns. We
  ;; ask s only for lengths that are remainders after an r-match in
  ;; 0..max(Ns). (And we call neither r nor s if there are no Ns.)
  (case (Ns.empty? none)
        (else
         (let r-matches (r (call set<- (range<- (+ (call max Ns) 1)))))
         (let Ns-r (call set<- (each '.count r-matches)))
         (let Ns-s (call set<- (for gather ((n Ns.keys))
                                 (for gather ((m Ns-r.keys))
                                   (let r (- n m))
                                   (if (< r 0) '() `(,r))))))
         (let s-matches (s Ns-s))
         (call set<- (for gather ((m1 r-matches))
                       (for gather ((m2 s-matches))
                         (if (Ns .maps? (+ m1.count m2.count))
                             `(,(chain m1 m2))
                             '())))))))

(let dot     (one-of "?"))
(let empty   (literal ""))
(to (plus r) (then r (star r)))

(export
  empty literal then either star plus maybe
  one-of dot nonempty
  generate)
<commit_msg>tweak<commit_after>;; Generate strings that match a regex.
;; From Udacity CS212 Python code by <NAME>.
;; Glossary:
;;    r, s   regex (i.e. a thing built by the constructors below)
;;    Ns     set of nonnegative integers (i.e. lengths)

;; Return the strings matching r whose length is in Ns.
(to (generate r Ns)
  (sort-by-key (r Ns)
               (given (str) `(,str.count ,str))))

(let none   (set<-))
(let just-0 (set<- 0))

(to (literal str)
  (let just-str (set<- str))
  (given (Ns)
    (if (Ns .maps? str.count) just-str none)))

(to (star r)
  (let r1 (nonempty r))
  (begin r1* ()
    (given (Ns) ((either empty (then r1 (r1*)))
                 Ns))))

(to ((nonempty r) Ns) (r (Ns .difference just-0)))
(to ((either r s) Ns) ((r Ns) .union (s Ns)))

(to (one-of chars)
  (let set (call set<- (as-list chars)))
  (given (Ns)
    (if (Ns .maps? 1) set null)))

(to ((then r s) Ns)
  ;; Return the set of matches to rs whose total length is in Ns. We
  ;; ask s only for lengths that are remainders after an r-match in
  ;; 0..max(Ns). (And we call neither r nor s if there are no Ns.)
  (case (Ns.empty? none)
        (else
         (let r-matches (r (call set<- (range<- (+ (call max Ns) 1)))))
         (let Ns-r (call set<- (each '.count r-matches)))
         (let Ns-s (call set<- (for gather ((n Ns.keys))
                                 (for gather ((m Ns-r.keys))
                                   (let r (- n m))
                                   (if (< r 0) '() `(,r))))))
         (let s-matches (s Ns-s))
         (call set<- (for gather ((m1 r-matches))
                       (for gather ((m2 s-matches))
                         (if (Ns .maps? (+ m1.count m2.count))
                             `(,(chain m1 m2))
                             '())))))))

(let dot      (one-of "?"))
(let empty    (literal ""))
(to (maybe r) (either empty r))
(to (plus r)  (then r (star r)))

(export
  empty literal then either star plus maybe
  one-of dot nonempty
  generate)

{'fraction_non_alphanumeric': 0.16842634489693314, 'fraction_numerical': 0.010055304172951232, 'mean_word_length': 1.8523297491039425, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2177307', 'n_tokens_mistral': 1450, 'n_tokens_neox': 1375, 'n_words': 531}