<commit_before>! Copyright (C) 2008 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: accessors arrays continuations deques dlists fry
io.directories io.files io.files.info io.pathnames kernel
sequences system vocabs.loader locals math namespaces
sorting assocs calendar threads io math.parser ;
IN: io.directories.search

: qualified-directory-entries ( path -- seq )
    dup directory-entries
    [ [ append-path ] change-name ] with map ;

: qualified-directory-files ( path -- seq )
    dup directory-files [ append-path ] with map ;

: with-qualified-directory-files ( path quot -- )
    '[ "" qualified-directory-files @ ] with-directory ; inline

: with-qualified-directory-entries ( path quot -- )
    '[ "" qualified-directory-entries @ ] with-directory ; inline

<PRIVATE

TUPLE: directory-iterator path bfs queue ;

: push-directory-entries ( path iter -- )
    [ [ qualified-directory-entries ] [ 2drop f ] recover ] dip '[
        _ [ queue>> ] [ bfs>> ] bi
        [ push-front ] [ push-back ] if
    ] each ;

: <directory-iterator> ( path bfs? -- iterator )
    <dlist> directory-iterator boa
    dup path>> over push-directory-entries ;

: next-directory-entry ( iter -- directory-entry/f )
    dup queue>> deque-empty? [ drop f ] [
        dup queue>> pop-back
        dup directory?
        [ name>> over push-directory-entries next-directory-entry ]
        [ nip ] if
    ] if ; recursive

:: iterate-directory-entries ( iter quot -- directory-entry/f )
    iter next-directory-entry [
        quot call( obj -- obj )
        [ iter quot iterate-directory-entries ] unless*
    ] [
        f
    ] if* ; inline recursive

: iterate-directory ( iter quot -- path/f )
    [ name>> ] prepose iterate-directory-entries ;

: setup-traversal ( path bfs quot -- iterator quot' )
    [ <directory-iterator> ] dip [ f ] compose ;

PRIVATE>

: each-file ( path bfs? quot -- )
    setup-traversal iterate-directory drop ;

: each-directory-entry ( path bfs? quot -- )
    setup-traversal iterate-directory-entries drop ;

: recursive-directory-files ( path bfs? -- paths )
    [ ] accumulator [ each-file ] dip ;

: recursive-directory-entries ( path bfs? -- paths )
    [ ] accumulator [ each-directory-entry ] dip ;

: find-file ( path bfs? quot -- path/f )
    [ <directory-iterator> ] dip
    [ keep and ] curry iterate-directory ;

: find-all-files ( path quot -- paths/f )
    [ f <directory-iterator> ] dip pusher
    [ [ f ] compose iterate-directory drop ] dip ;

ERROR: file-not-found path bfs? quot ;

: find-file-throws ( path bfs? quot -- path )
    3dup find-file dup [ 2nip nip ] [ drop file-not-found ] if ;

: find-in-directories ( directories bfs? quot -- path'/f )
    '[ _ [ _ _ find-file-throws ] attempt-all ]
    [ drop f ] recover ;

: find-all-in-directories ( directories quot -- paths/f )
    '[ _ find-all-files ] map concat ;

: link-size/0 ( path -- n )
    [ link-info size-on-disk>> ] [ 2drop 0 ] recover ;

: directory-size ( path -- n )<commit_msg>fix help-lint, compilation issue in io.directories.search<commit_after>! Copyright (C) 2008 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: accessors arrays continuations deques dlists fry
io.directories io.files io.files.info io.pathnames kernel
sequences system vocabs.loader locals math namespaces
sorting assocs calendar threads io math.parser ;
IN: io.directories.search

: qualified-directory-entries ( path -- seq )
    dup directory-entries
    [ [ append-path ] change-name ] with map ;

: qualified-directory-files ( path -- seq )
    dup directory-files [ append-path ] with map ;

: with-qualified-directory-files ( path quot -- )
    '[ "" qualified-directory-files @ ] with-directory ; inline

: with-qualified-directory-entries ( path quot -- )
    '[ "" qualified-directory-entries @ ] with-directory ; inline

<PRIVATE

TUPLE: directory-iterator path bfs queue ;

: push-directory-entries ( path iter -- )
    [ [ qualified-directory-entries ] [ 2drop f ] recover ] dip '[
        _ [ queue>> ] [ bfs>> ] bi
        [ push-front ] [ push-back ] if
    ] each ;

: <directory-iterator> ( path bfs? -- iterator )
    <dlist> directory-iterator boa
    dup path>> over push-directory-entries ;

: next-directory-entry ( iter -- directory-entry/f )
    dup queue>> deque-empty? [ drop f ] [
        dup queue>> pop-back
        dup directory?
        [ name>> over push-directory-entries next-directory-entry ]
        [ nip ] if
    ] if ;

:: iterate-directory-entries ( iter quot: ( obj -- obj ) -- directory-entry/f )
    iter next-directory-entry [
        quot call
        [ iter quot iterate-directory-entries ] unless*
    ] [
        f
    ] if* ; inline recursive

: iterate-directory ( iter quot -- path/f )
    [ name>> ] prepose iterate-directory-entries ; inline

: setup-traversal ( path bfs quot -- iterator quot' )
    [ <directory-iterator> ] dip [ f ] compose ; inline

PRIVATE>

: each-file ( path bfs? quot -- )
    setup-traversal iterate-directory drop ; inline

: each-directory-entry ( path bfs? quot -- )
    setup-traversal iterate-directory-entries drop ; inline

: recursive-directory-files ( path bfs? -- paths )
    [ ] accumulator [ each-file ] dip ; inline

: recursive-directory-entries ( path bfs? -- directory-entries )
    [ ] accumulator [ each-directory-entry ] dip ; inline

: find-file ( path bfs? quot -- path/f )
    [ <directory-iterator> ] dip
    [ keep and ] curry iterate-directory ; inline

: find-all-files ( path quot -- paths/f )
    [ f <directory-iterator> ] dip pusher
    [ [ f ] compose iterate-directory drop ] dip ; inline

ERROR: file-not-found path bfs? quot ;

: find-file-throws ( path bfs? quot -- path )
    3dup find-file dup [ 2nip nip ] [ drop file-not-found ] if ; inline

: find-in-directories ( directories bfs? quot -- path'/f )
    '[ _ [ _ _ find-file-throws ] attempt-all ]
    [ drop f ] recover ; inline

: find-all-in-directories ( directories quot -- paths/f )
    '[ _ find-all-files ] map concat ; inline

: link-size/0 ( path -- n )
    [ link-info size-on-disk>> ] [ 2drop 0 ] recover ;

: directory-size ( path -- n )
{'fraction_non_alphanumeric': 0.13164310092637738, 'fraction_numerical': 0.0032504469364537623, 'mean_word_length': 2.6696481812760884, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3387589', 'n_tokens_mistral': 1934, 'n_tokens_neox': 1872, 'n_words': 519}