<commit_before>;; Ported from: _Metamagical Themas_ by <NAME>, 

;; N.B. expressing probabilities as a fraction of 1000, since we don't yet
;; do floating point numbers.

(import (use "lib/parson")    parse grammar<-)
(import (use "lib/text-wrap") fill)

(import (use "lib/random") rng)         ;TODO pass it in

(to (main `(,_ ,@words))
  (let expanded (expand-text (" " .join words) 1000))
  (format "~d\n" (fill expanded 72)))

(to (expand-text text probability)
  (call chain
        (for each ((token ((parse parser text) .opt-results)))
          (expand token probability))))

(let grammar (grammar<- "
phrase: (word | :anyone)*.
word:   :letter+ {'!'}? :join.
"))
(let parser ((grammar (map<-)) "phrase"))

(to (expand token probability)
  (if (and (menu .maps? token) (< (rng .random-integer 1000) probability))
      (expand-text (menu token) (/ (* probability 4) 5))
      token))

(let menu
  (map<-
   '(("tomatoes" "tomatoes on macaroni (and tomatoes only), exquisitely spiced")
     ("macaroni" "macaroni and cheese (a repast of Naples, Italy)")
     ("repast" "rather extraordinary pasta and sauce, typical")
     ("cheese" "cheddar, havarti, Emmentaler (particularly sharp Emmentaler)")<commit_msg>tweak<commit_after>;; Ported from: _Metamagical Themas_ by <NAME>, 

;; N.B. expressing probabilities as a fraction of 1000, since we don't yet
;; do floating point numbers.

(import (use "lib/parson")    parse grammar<-)
(import (use "lib/text-wrap") fill)

(import (use "lib/random") rng)         ;TODO pass it in

(to (main `(,_ ,@words))
  (let expanded (expand-text (" " .join words) 1000))
  (format "~d\n" (fill expanded 72)))

(to (expand-text text probability)
  (call chain (for each ((token ((parse parser text) .opt-results)))
                (expand token probability))))

(let grammar (grammar<- "
phrase: (word | :anyone)*.
word:   :letter+ {'!'}? :join.
"))
(let parser ((grammar (map<-)) "phrase"))

(to (expand token probability)
  (if (and (menu .maps? token) (< (rng .random-integer 1000) probability))
      (expand-text (menu token) (/ (* probability 4) 5))
      token))

(let menu
  (map<-
   '(("tomatoes" "tomatoes on macaroni (and tomatoes only), exquisitely spiced")
     ("macaroni" "macaroni and cheese (a repast of Naples, Italy)")
     ("repast" "rather extraordinary pasta and sauce, typical")
     ("cheese" "cheddar, havarti, Emmentaler (particularly sharp Emmentaler)")
{'fraction_non_alphanumeric': 0.16611570247933885, 'fraction_numerical': 0.013223140495867768, 'mean_word_length': 3.775147928994083, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7267985', 'n_tokens_mistral': 859, 'n_tokens_neox': 781, 'n_words': 277}