<commit_before>! Copyright (C) 2008, 2010 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: accessors command-line continuations fry init io
io.backend io.files io.monitors io.pathnames kernel namespaces
prettyprint sequences splitting threads tr vocabs vocabs.cache
vocabs.loader vocabs.refresh ;
IN: vocabs.refresh.monitor

TR: convert-separators "/\\" ".." ;

: vocab-dir>vocab-name ( path -- vocab )
    trim-head-separators
    trim-tail-separators
    convert-separators ;

: path>vocab-name ( path -- vocab )
    dup ".factor" tail? [ parent-directory ] when ;

: chop-vocab-root ( path -- path' )
    "resource:" prepend-path normalize-path
    dup vocab-roots get
    [ [ normalize-path ] map ]
    [ [ resolve-symlinks ] map ] bi append
    [ head? ] with find nip
    ?head drop ;

: path>vocab ( path -- vocab )
    chop-vocab-root path>vocab-name vocab-dir>vocab-name ;

: monitor-loop ( monitor -- )
    ! On OS X, monitors give us the full path, so we chop it
    ! off if its there.
    [
        next-change path>> path>vocab
        [ changed-vocab ] [ reset-cache ] bi
    ]
    [ monitor-loop ]
    bi ;

: (start-vocab-monitor) ( vocab-root -- )
    dup exists?
    [ [ t <monitor> monitor-loop ] with-monitors ] [ drop ] if ;

: start-vocab-monitor ( vocab-root -- )
    [
        dup '[
            [ _ (start-vocab-monitor) ]
            [ [ _ "fatal error for monitor root ``" "``: " surround write . flush ] with-global ] recover
        ]
    ] [ "Root monitor: " prepend ]
    bi spawn drop ;

: init-vocab-monitor ( -- )
    H{ } clone changed-vocabs set-global
    loaded-vocab-names [ changed-vocab ] each ;

[
    "-no-monitors" (command-line) member? [
        [ drop ] add-vocab-root-hook set-global
        f changed-vocabs set-global<commit_msg>vocabs.refresh.monitor: Add exception handling in monitor-loop so that if a path isn't a valid vocabulary name then we print an error. Should spam the console on windows downloads but 1) only for .com not the .exe, 2) we prefer people to build from source in which case the :Zone.Identifier ADS will not be present, and 3) it will educate people about ADS hopefully.<commit_after>! Copyright (C) 2008, 2010 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: accessors command-line continuations fry init io
io.backend io.files io.monitors io.pathnames kernel namespaces
prettyprint sequences splitting threads tr vocabs vocabs.cache
vocabs.loader vocabs.refresh ;
IN: vocabs.refresh.monitor

TR: convert-separators "/\\" ".." ;

: vocab-dir>vocab-name ( path -- vocab )
    trim-head-separators
    trim-tail-separators
    convert-separators ;

: path>vocab-name ( path -- vocab )
    dup ".factor" tail? [ parent-directory ] when ;

: chop-vocab-root ( path -- path' )
    "resource:" prepend-path normalize-path
    dup vocab-roots get
    [ [ normalize-path ] map ]
    [ [ resolve-symlinks ] map ] bi append
    [ head? ] with find nip
    ?head drop ;

: path>vocab ( path -- vocab )
    chop-vocab-root path>vocab-name vocab-dir>vocab-name ;

: monitor-loop ( monitor -- )
    ! On OS X, monitors give us the full path, so we chop it
    ! off if its there.
    [
        next-change path>>
        [
            path>vocab
            [ changed-vocab ] [ reset-cache ] bi
        ] [
            [
                [ "monitor-loop warning for path ``" "``:" surround write ]
                [ . ] bi* flush
            ] with-global
        ] recover
    ] [ monitor-loop ] bi ;

: (start-vocab-monitor) ( vocab-root -- )
    dup exists?
    [ [ t <monitor> monitor-loop ] with-monitors ] [ drop ] if ;

: start-vocab-monitor ( vocab-root -- )
    [
        dup '[
            [ _ (start-vocab-monitor) ]
            [
                [
                    _ "fatal error for monitor root ``" "``: " surround write
                    . flush
                ] with-global
            ] recover
        ]
    ] [ "Root monitor: " prepend ]
    bi spawn drop ;

: init-vocab-monitor ( -- )
    H{ } clone changed-vocabs set-global
    loaded-vocab-names [ changed-vocab ] each ;

[
    "-no-monitors" (command-line) member? [
        [ drop ] add-vocab-root-hook set-global
        f changed-vocabs set-global
{'fraction_non_alphanumeric': 0.12196272705826847, 'fraction_numerical': 0.004482189195564991, 'mean_word_length': 2.2048374905517765, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4430163', 'n_tokens_mistral': 1403, 'n_tokens_neox': 1357, 'n_words': 375}