<commit_before>! Copyright (C) 2007, 2008 <NAME>, <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: namespaces sequences io.files kernel assocs words vocabs
definitions parser continuations io hashtables sorting
source-files arrays combinators strings system math.parser
compiler.errors splitting init accessors ;
IN: vocabs.loader

SYMBOL: vocab-roots

V{
    "resource:core"
    "resource:basis"
    "resource:extra"
    "resource:work"
} clone vocab-roots set-global

: vocab-dir ( vocab -- dir )
    vocab-name { { CHAR: . CHAR: / } } substitute ;

: vocab-dir+ ( vocab str/f -- path )
    >r vocab-name "." split r>
    [ >r dup peek r> append suffix ] when*
    "/" join ;

: vocab-dir? ( root name -- ? )
    over [
        ".factor" vocab-dir+ append-path exists?
    ] [
        2drop f
    ] if ;

SYMBOL: root-cache

H{ } clone root-cache set-global

: (find-vocab-root) ( name -- path/f )
    vocab-roots get swap [ vocab-dir? ] curry find nip ;

: find-vocab-root ( vocab -- path/f )
    vocab-name dup root-cache get at [ ] [ (find-vocab-root) ] ?if ;

: vocab-append-path ( vocab path -- newpath )
    swap find-vocab-root dup [ prepend-path ] [ 2drop f ] if ;

: vocab-source-path ( vocab -- path/f )
    dup ".factor" vocab-dir+ vocab-append-path ;

: vocab-docs-path ( vocab -- path/f )
    dup "-docs.factor" vocab-dir+ vocab-append-path ;

SYMBOL: load-help?

: load-source ( vocab -- vocab )
    f over set-vocab-source-loaded?
    [ vocab-source-path [ parse-file ] [ [ ] ] if* ] keep
    t over set-vocab-source-loaded?
    [ [ % ] [ call ] if-bootstrapping ] dip ;


: load-docs ( vocab -- vocab )
    load-help? get [
        f over set-vocab-docs-loaded?
        [ vocab-docs-path [ ?run-file ] when* ] keep
        t over set-vocab-docs-loaded?
    ] when ;

: reload ( name -- )
    [
        dup vocab [ load-source load-docs drop ] [ no-vocab ] ?if
    ] with-compiler-errors ;

: require ( vocab -- )
    load-vocab drop ;

: run ( vocab -- )
    dup load-vocab vocab-main [
        execute
    ] [
        "The " write vocab-name write
        " vocabulary does not define an entry point." print
        "To define one, refer to \\ MAIN: help" print
    ] ?if ;

SYMBOL: blacklist

: add-to-blacklist ( error vocab -- )
    vocab-name blacklist get dup [ set-at ] [ 3drop ] if ;

GENERIC: (load-vocab) ( name -- )<commit_msg>Remove blank line<commit_after>! Copyright (C) 2007, 2008 <NAME>, <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: namespaces sequences io.files kernel assocs words vocabs
definitions parser continuations io hashtables sorting
source-files arrays combinators strings system math.parser
compiler.errors splitting init accessors ;
IN: vocabs.loader

SYMBOL: vocab-roots

V{
    "resource:core"
    "resource:basis"
    "resource:extra"
    "resource:work"
} clone vocab-roots set-global

: vocab-dir ( vocab -- dir )
    vocab-name { { CHAR: . CHAR: / } } substitute ;

: vocab-dir+ ( vocab str/f -- path )
    >r vocab-name "." split r>
    [ >r dup peek r> append suffix ] when*
    "/" join ;

: vocab-dir? ( root name -- ? )
    over [
        ".factor" vocab-dir+ append-path exists?
    ] [
        2drop f
    ] if ;

SYMBOL: root-cache

H{ } clone root-cache set-global

: (find-vocab-root) ( name -- path/f )
    vocab-roots get swap [ vocab-dir? ] curry find nip ;

: find-vocab-root ( vocab -- path/f )
    vocab-name dup root-cache get at [ ] [ (find-vocab-root) ] ?if ;

: vocab-append-path ( vocab path -- newpath )
    swap find-vocab-root dup [ prepend-path ] [ 2drop f ] if ;

: vocab-source-path ( vocab -- path/f )
    dup ".factor" vocab-dir+ vocab-append-path ;

: vocab-docs-path ( vocab -- path/f )
    dup "-docs.factor" vocab-dir+ vocab-append-path ;

SYMBOL: load-help?

: load-source ( vocab -- vocab )
    f over set-vocab-source-loaded?
    [ vocab-source-path [ parse-file ] [ [ ] ] if* ] keep
    t over set-vocab-source-loaded?
    [ [ % ] [ call ] if-bootstrapping ] dip ;

: load-docs ( vocab -- vocab )
    load-help? get [
        f over set-vocab-docs-loaded?
        [ vocab-docs-path [ ?run-file ] when* ] keep
        t over set-vocab-docs-loaded?
    ] when ;

: reload ( name -- )
    [
        dup vocab [ load-source load-docs drop ] [ no-vocab ] ?if
    ] with-compiler-errors ;

: require ( vocab -- )
    load-vocab drop ;

: run ( vocab -- )
    dup load-vocab vocab-main [
        execute
    ] [
        "The " write vocab-name write
        " vocabulary does not define an entry point." print
        "To define one, refer to \\ MAIN: help" print
    ] ?if ;

SYMBOL: blacklist

: add-to-blacklist ( error vocab -- )
    vocab-name blacklist get dup [ set-at ] [ 3drop ] if ;

GENERIC: (load-vocab) ( name -- )
{'fraction_non_alphanumeric': 0.14072673808023525, 'fraction_numerical': 0.004620877966813695, 'mean_word_length': 2.334733893557423, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2992185', 'n_tokens_mistral': 1736, 'n_tokens_neox': 1694, 'n_words': 448}