<commit_before>! Copyright (C) 2005, 2009 <NAME>.
! See http://factorcode.org/license.txt for BSD license.

USING: accessors arrays assocs combinators fry io kernel locals
make math math.order namespaces sequences sorting strings
unicode.case unicode.categories unicode.data vectors vocabs
vocabs.hierarchy words ;

IN: tools.completion

:: (fuzzy) ( accum i full ch -- accum i full ? )
    ch i full index-from [
        :> i i accum push
        accum i 1 + full t
    ] [
        f -1 full f
    ] if* ;

: fuzzy ( full short -- indices )
    dup [ length <vector> 0 ] curry 2dip
    [ (fuzzy) ] all? 3drop ;

: (runs) ( runs n seq -- runs n )
    [
        [
            2dup number=
            [ drop ] [ nip V{ } clone pick push ] if
            1 +
        ] keep pick last push
    ] each ;

: runs ( seq -- newseq )
    V{ V{ } } [ clone ] map over first rot (runs) drop ;

: score-1 ( i full -- n )
    {
        { [ over zero? ] [ 2drop 10 ] }
        { [ 2dup length 1 - number= ] [ 2drop 4 ] }
        { [ 2dup [ 1 - ] dip nth Letter? not ] [ 2drop 10 ] }
        { [ 2dup [ 1 + ] dip nth Letter? not ] [ 2drop 4 ] }
        [ 2drop 1 ]
    } cond ;

: score ( full fuzzy -- n )
    dup [
        [ [ length ] bi@ - 15 swap [-] 3 /f ] 2keep
        runs [
            [ 0 [ pick score-1 max ] reduce nip ] keep
            length * +
        ] with each
    ] [
        2drop 0
    ] if ;

: rank-completions ( results -- newresults )
    sort-keys <reversed>
    [ 0 [ first max ] reduce 3 /f ] keep
    [ first < ] with filter
    values ;

: complete ( full short -- score )
    [ dupd fuzzy score ] 2keep
    [ <reversed> ] bi@
    dupd fuzzy score max ;

: completion ( short candidate -- result )
    [ second >lower swap complete ] keep 2array ;

: completion, ( short candidate -- )
    completion dup first 0 > [ , ] [ drop ] if ;

: completions ( short candidates -- seq )
    [ ] [
        [ >lower ] dip [ [ completion, ] with each ] { } make
        rank-completions
    ] bi-curry if-empty ;

: name-completions ( str seq -- seq' )
    [ dup name>> ] { } map>assoc completions ;<commit_msg>tools.completion: speed up apropos by 35%.  Fixes #101.<commit_after>! Copyright (C) 2005, 2009 <NAME>.
! See http://factorcode.org/license.txt for BSD license.

USING: accessors arrays assocs combinators fry io kernel locals
make math math.order namespaces sequences sorting strings
unicode.case unicode.categories unicode.data vectors vocabs
vocabs.hierarchy words ;

IN: tools.completion

<PRIVATE

: smart-index-from ( obj i seq -- n/f )
    rot [ ch>lower ] [ ch>upper ] bi
    [ eq? ] bi-curry@ [ bi or ] 2curry find-from drop ;

:: (fuzzy) ( accum i full ch -- accum i full ? )
    ch i full smart-index-from [
        :> i i accum push
        accum i 1 + full t
    ] [
        f -1 full f
    ] if* ;

: fuzzy ( full short -- indices )
    dup [ length <vector> 0 ] curry 2dip
    [ (fuzzy) ] all? 3drop ;

: (runs) ( runs n seq -- runs n )
    [
        [
            2dup number=
            [ drop ] [ nip V{ } clone pick push ] if
            1 +
        ] keep pick last push
    ] each ;

: runs ( seq -- newseq )
    V{ V{ } } [ clone ] map over first rot (runs) drop ;

: score-1 ( i full -- n )
    {
        { [ over zero? ] [ 2drop 10 ] }
        { [ 2dup length 1 - number= ] [ 2drop 4 ] }
        { [ 2dup [ 1 - ] dip nth Letter? not ] [ 2drop 10 ] }
        { [ 2dup [ 1 + ] dip nth Letter? not ] [ 2drop 4 ] }
        [ 2drop 1 ]
    } cond ;

: score ( full fuzzy -- n )
    dup [
        [ [ length ] bi@ - 15 swap [-] 3 /f ] 2keep
        runs [
            [ 0 [ pick score-1 max ] reduce nip ] keep
            length * +
        ] with each
    ] [
        2drop 0
    ] if ;

: rank-completions ( results -- newresults )
    sort-keys <reversed>
    [ 0 [ first max ] reduce 3 /f ] keep
    [ first < ] with filter
    values ;

: complete ( full short -- score )
    [ dupd fuzzy score ] 2keep
    [ <reversed> ] bi@
    dupd fuzzy score max ;

: completion ( short candidate -- result )
    [ second swap complete ] keep 2array ;

: completion, ( short candidate -- )
    completion dup first 0 > [ , ] [ drop ] if ;

PRIVATE>

: completions ( short candidates -- seq )
    [ ] [
        [ [ completion, ] with each ] { } make
        rank-completions
    ] bi-curry if-empty ;

: name-completions ( str seq -- seq' )
    [ dup name>> ] { } map>assoc completions ;
{'fraction_non_alphanumeric': 0.125, 'fraction_numerical': 0.022644927536231884, 'mean_word_length': 1.7797356828193833, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1069891', 'n_tokens_mistral': 1565, 'n_tokens_neox': 1430, 'n_words': 496}