<commit_before>! Copyright (C) 2007, 2010 <NAME>, <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: accessors arrays assocs continuations definitions init
io io.files io.pathnames kernel make namespaces parser
sequences sets splitting strings vocabs words ;
IN: vocabs.loader

SYMBOL: vocab-roots

SYMBOL: add-vocab-root-hook

[
    V{
        "resource:core"
        "resource:basis"
        "resource:extra"
        "resource:work"
    } clone vocab-roots set-global

    [ drop ] add-vocab-root-hook set-global
] "vocabs.loader" add-startup-hook

: add-vocab-root ( root -- )
    trim-tail-separators dup vocab-roots get ?adjoin
    [ add-vocab-root-hook get-global call( root -- ) ] [ drop ] if ;

SYMBOL: root-cache
root-cache [ H{ } clone ] initialize

ERROR: not-found-in-roots path ;

<PRIVATE

: find-root-for ( path -- path/f )
    vocab-roots get [ prepend-path exists? ] with find nip ;

M: string vocab-path ( string -- path/f )
    dup find-root-for [ prepend-path ] [ not-found-in-roots ] if* ;
<commit_msg>vocabs.loader: make a word for the default vocabs.<commit_after>! Copyright (C) 2007, 2010 <NAME>, <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: accessors arrays assocs continuations definitions init
io io.files io.pathnames kernel make namespaces parser
sequences sets splitting strings vocabs words ;
IN: vocabs.loader

SYMBOL: vocab-roots

SYMBOL: add-vocab-root-hook

CONSTANT: default-vocab-roots {
    "resource:core"
    "resource:basis"
    "resource:extra"
    "resource:work"
}

[
    default-vocab-roots V{ } like vocab-roots set-global

    [ drop ] add-vocab-root-hook set-global
] "vocabs.loader" add-startup-hook

: add-vocab-root ( root -- )
    trim-tail-separators dup vocab-roots get ?adjoin
    [ add-vocab-root-hook get-global call( root -- ) ] [ drop ] if ;

SYMBOL: root-cache
root-cache [ H{ } clone ] initialize

ERROR: not-found-in-roots path ;

<PRIVATE

: find-root-for ( path -- path/f )
    vocab-roots get [ prepend-path exists? ] with find nip ;

M: string vocab-path ( string -- path/f )
    dup find-root-for [ prepend-path ] [ not-found-in-roots ] if* ;

{'fraction_non_alphanumeric': 0.12691060676239, 'fraction_numerical': 0.007410838351088467, 'mean_word_length': 3.16988416988417, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1188309', 'n_tokens_mistral': 776, 'n_tokens_neox': 739, 'n_words': 198}