<commit_before>#lang racket/base
(require racket/contract/base
         syntax/modcollapse
         setup/main-collects
         scribble/core
         ;; Needed to normalize planet version numbers:
         (only-in planet/resolver get-planet-module-path/pkg)
         (only-in planet/private/data pkg-maj pkg-min))

(provide
 (contract-out
  [make-section-tag ((string?) 
                     (#:doc (or/c #f module-path?)
                      #:tag-prefixes (or/c #f (listof string?)))
                     . ->* .
                     tag?)]
  [make-module-language-tag (-> symbol? tag?)]
  [taglet? (any/c . -> . boolean?)]
  [module-path-prefix->string (module-path? . -> . string?)]
  [module-path-index->taglet (module-path-index? . -> . taglet?)]
  [intern-taglet (any/c . -> . any/c)]
  [doc-prefix (case->
               ((or/c #f module-path?) taglet? . -> . taglet?)
               ((or/c #f module-path?) (or/c #f (listof string?)) taglet? . -> . taglet?))]))

(define (make-section-tag s #:doc [doc #f] #:tag-prefixes [prefix #f])
  `(part ,(doc-prefix doc prefix s)))

(define (make-module-language-tag langname)
  `(mod-path ,(symbol->string langname)))

(define (taglet? v)
  (and (not (generated-tag? v))
       (tag? (list 'something v))))

(define interned (make-weak-hash))
  
(define (intern-taglet v)
  (let ([v (if (list? v)
               (map intern-taglet v)
               (datum-intern-literal v))])
    (if (or (string? v)
            (bytes? v)
            (list? v))
        (let ([b (hash-ref interned v #f)])
          (if b<commit_msg>scibble: repair related to recent `scribble/tag' contracts<commit_after>#lang racket/base
(require racket/contract/base
         syntax/modcollapse
         setup/main-collects
         scribble/core
         ;; Needed to normalize planet version numbers:
         (only-in planet/resolver get-planet-module-path/pkg)
         (only-in planet/private/data pkg-maj pkg-min))

(provide
 (contract-out
  [make-section-tag ((string?) 
                     (#:doc (or/c #f module-path?)
                      #:tag-prefixes (or/c #f (listof string?)))
                     . ->* .
                     tag?)]
  [make-module-language-tag (-> symbol? tag?)]
  [taglet? (any/c . -> . boolean?)]
  [module-path-prefix->string (module-path? . -> . string?)]
  [module-path-index->taglet (module-path-index? . -> . taglet?)]
  [intern-taglet (any/c . -> . any/c)]
  [doc-prefix (case->
               ((or/c #f module-path?) taglet? . -> . taglet?)
               ((or/c #f module-path?) (or/c #f (listof string?)) taglet? . -> . taglet?))]))

(define (make-section-tag s #:doc [doc #f] #:tag-prefixes [prefix #f])
  `(part ,(doc-prefix doc prefix s)))

(define (make-module-language-tag langname)
  `(mod-path ,(symbol->string langname)))

(define (taglet? v)
  (and (not (generated-tag? v))
       (tag? (list 'something (list v)))))

(define interned (make-weak-hash))
  
(define (intern-taglet v)
  (let ([v (if (list? v)
               (map intern-taglet v)
               (datum-intern-literal v))])
    (if (or (string? v)
            (bytes? v)
            (list? v))
        (let ([b (hash-ref interned v #f)])
          (if b
{'fraction_non_alphanumeric': 0.1802252816020025, 'fraction_numerical': 0.0, 'mean_word_length': 2.0418648905803995, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '86929', 'n_tokens_mistral': 1120, 'n_tokens_neox': 1085, 'n_words': 262}