<commit_before>#lang scheme/base
(require (prefix-in doc: scribble/doc/reader))
(provide (rename-out [doc:read read] [doc:read-syntax read-syntax])
         get-info)

(define (get-info . args)
  (lambda (key defval)
    (case key
      [(color-lexer)
       (dynamic-require 'syntax-color/scribble-lexer 'scribble-inside-lexer)]
      [(drscheme:toolbar-buttons)
       (dynamic-require 'scribble/tools/drscheme-buttons 'drscheme-buttons)]
      [else defval])))<commit_msg>fix module-laguage info from scribble/doc<commit_after>#lang scheme/base
(require (prefix-in doc: scribble/doc/reader))
(provide (rename-out [doc:read read] 
                     [my:read-syntax read-syntax])
         get-info)

(define (my:read-syntax . args)
  (let ([s (apply doc:read-syntax args)])
    ;; For now, remove the 'module-language property added by `doc:read-syntax'
    (syntax-property s 'module-language #f)))

(define (get-info . args)
  (lambda (key defval)
    (case key
      [(color-lexer)
       (dynamic-require 'syntax-color/scribble-lexer 'scribble-inside-lexer)]
      [(drscheme:toolbar-buttons)
       (dynamic-require 'scribble/tools/drscheme-buttons 'drscheme-buttons)]
      [else defval])))
{'fraction_non_alphanumeric': 0.14666666666666667, 'fraction_numerical': 0.0, 'mean_word_length': 3.383211678832117, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '507115', 'n_tokens_mistral': 416, 'n_tokens_neox': 392, 'n_words': 86}