<commit_before>(load "interpreter.scm")

; Example
(define load-result (load-words "vocabulary/scrape.scm"))
(define test-vocabulary (car load-result))
(define test-pos-table (cdr load-result))
(define test-interpreter (make-interpreter test-vocabulary dummy-grader))
(define test-word (match-word 'a (number-syllables 2)
                                 (rhymes-with "clicker")))
(define test-line (match-line 'b "more-literals"
                                 test-word
                                 (match-word (rhymes-with "corruption"))))
(define test-constraints (poem
                           (match-line 'a
                                       ;'(syllables 10)
                                       (match-word 'y (has-synonym "learning"))
                                       (match-word 'x (rhymes-with "investing")))
                           (match-line test-word)
                           (match-line '(syllables 6)
                                       (match-word (any-word)))
                           (match-line '(syllables 10)
                                       (match-word (any-word))
                                       (match-word (any-word)))
                           (match-line 'a)
                           (match-line (match-word 'x)
                                       (match-word 'a))
                           test-line))
(define (test)
  (let ((test-poem (test-interpreter test-constraints)))
    test-poem))
<commit_msg>Add better demos.<commit_after>(load "interpreter.scm")

; Example
(define load-result (load-words "vocabulary/scrape.scm"))
(define vocab (car load-result))
(define pos-table (cdr load-result))
(define interpreter (make-interpreter vocab random-choice))

(define test-haiku (poem
                    (match-line 'a
                                '(syllables 5)
                                (match-word (any-word))
                                (match-word (any-word)))
                    (match-line 'b
                                '(syllables 7)
                                (match-word (rhymes-with "allay"))
                                (match-word (any-word)))
                    (match-line 'c
                                '(syllables 5)
                                (match-word (any-word))
                                (match-word (any-word)))))
#|
(print-poem (interpreter test-haiku))
; sauciness bayoneted
; crochet honoraria
; suffused overtook
|#

(define test-poem (poem
                    (match-line 'a
                                ;'(syllables 10)
                                (match-word 'y (has-synonym "learning"))
                                (match-word 'x (rhymes-with "investing")))
                    (match-line '(syllables 6)
                                (match-word (any-word)))
                    (match-line '(syllables 10)
                                (match-word (any-word))
                                (match-word (any-word)))
                    (match-line 'a)
                    (match-line (match-word 'x)
                                (match-word 'a))))
#|
(print-poem (interpreter test-poem))
; unlearning vesting
; mariposas
; dirigibles mariposas
; unlearning vesting
; vesting enunciates
|#

{'fraction_non_alphanumeric': 0.11516079632465544, 'fraction_numerical': 0.004287901990811639, 'mean_word_length': 1.1180285343709468, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4162235', 'n_tokens_mistral': 915, 'n_tokens_neox': 869, 'n_words': 186}