<commit_before>#lang racket
;; takes all diretories given in the test submodule
;; for every .rkt file in those directories it loads
;; tests that file and checks its coverage against an
;; .rktl file of the same name
(require better-test racket/runtime-path rackunit)

(define (test-dir d)
  (define files
    (for*/list ([p (directory-list d)]
                [s (in-value (path->string (build-path d p)))]
                #:when (regexp-match #rx"\\.rkt$" s))
      s))
  (define covered
    (for/hash ([f files])
      (values f
              (path->string (path-replace-suffix f ".rktl")))))

  (define (do-test files)
    (apply test-files! files)

    (define coverage (get-test-coverage))
    (for ([(program cover) covered])
      (define actual-coverage (hash-ref coverage program))
      (define-values (expected-coverage expected-uncoverage)
        (with-input-from-file cover (lambda () (values (ranges->numbers (read))
                                                       (ranges->numbers (read))))))
      (test-begin
       (for ([i expected-coverage])
         (check-true (covered? i actual-coverage)
                     (format "expected char ~a to be covered, but it was not, in: ~s"
                             i program)))
       (for ([i expected-uncoverage])
         (check-true (not (covered? i actual-coverage))
                     (format "expected char ~a to be uncovered, but it was, in: ~s"<commit_msg>fixed build failure on first install<commit_after>#lang racket
;; takes all diretories given in the test submodule
;; for every .rkt file in those directories it loads
;; tests that file and checks its coverage against an
;; .rktl file of the same name
(require "../main.rkt" racket/runtime-path rackunit)

(define (test-dir d)
  (define files
    (for*/list ([p (directory-list d)]
                [s (in-value (path->string (build-path d p)))]
                #:when (regexp-match #rx"\\.rkt$" s))
      s))
  (define covered
    (for/hash ([f files])
      (values f
              (path->string (path-replace-suffix f ".rktl")))))

  (define (do-test files)
    (apply test-files! files)

    (define coverage (get-test-coverage))
    (for ([(program cover) covered])
      (define actual-coverage (hash-ref coverage program))
      (define-values (expected-coverage expected-uncoverage)
        (with-input-from-file cover (lambda () (values (ranges->numbers (read))
                                                       (ranges->numbers (read))))))
      (test-begin
       (for ([i expected-coverage])
         (check-true (covered? i actual-coverage)
                     (format "expected char ~a to be covered, but it was not, in: ~s"
                             i program)))
       (for ([i expected-uncoverage])
         (check-true (not (covered? i actual-coverage))
                     (format "expected char ~a to be uncovered, but it was, in: ~s"
{'fraction_non_alphanumeric': 0.12986565621770582, 'fraction_numerical': 0.0, 'mean_word_length': 2.14626218851571, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '295753', 'n_tokens_mistral': 859, 'n_tokens_neox': 822, 'n_words': 286}