<commit_before>! Copyright (C) 2005, 2009 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: accessors arrays ascii combinators
combinators.short-circuit compiler.units definitions help
help.markup help.topics kernel lexer math namespaces parser
sequences splitting strings vocabs.parser words ;
IN: help.syntax

<PRIVATE

:: parse-help-token ( -- str/obj/f )
    ?scan-token dup search {
        { [ dup \ } eq? ] [ 2drop f ] }
        { [ dup parsing-word? ] [
            nip V{ } clone swap execute-parsing first
            dup wrapper? [ wrapped>> \ $link swap 2array ] when ] }
        [ drop ]
    } cond ;

: push-help-text ( accum sbuf obj -- accum sbuf' )
    [ dup empty? [ >string suffix! SBUF" " clone ] unless ]
    [ [ suffix! ] curry dip ] bi* ;

: push-help-space ( accum sbuf -- accum sbuf )
    {
        [ dup empty? not ]
        [ over empty? not pick ?last \ $nl eq? not and ]
    } 0|| [ CHAR: \s suffix! ] when ;

:: parse-help-text ( -- seq )
    V{ } clone SBUF" " clone [
        lexer get line>> parse-help-token [
            lexer get line>> swap - 1 > [
                \ $nl push-help-text
            ] when
        ] dip [
            [
                dup string? [
                    dup ?first ".,;:" member? [
                        [ push-help-space ] dip
                    ] unless append!
                ] [
                    [ push-help-space ]
                    [ push-help-text ] bi*
                ] if
            ] when*
        ] keep
    ] loop [ >string suffix! ] unless-empty >array ; inline

: parse-help-values ( -- seq )
    [ scan-token dup "}" = not ] [
        dup "{" = [
            parse-datum dup parsing-word?
            [ V{ } clone swap execute-parsing first ] when
        ] [
            ":" ?tail drop scan-object 2array
        ] if
    ] produce nip ;

: example-lines ( seq -- seq' )
    dup string? [ string-lines [ [ blank? ] trim ] map harvest ] when ;

: make-example ( str type -- seq )
    over string? [
        [ example-lines ] [ prefix ] bi*
    ] [ drop ] if ;

: parse-help-examples ( -- seq )
    \ } parse-until [ \ $example make-example ] { } map-as ;

: parse-help-example ( -- seq )<commit_msg>help.syntax: parse-help-values needs to parse-array directly.<commit_after>! Copyright (C) 2005, 2009 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: accessors arrays ascii combinators
combinators.short-circuit compiler.units definitions help
help.markup help.topics kernel lexer math namespaces parser
sequences splitting strings vocabs.parser words ;
IN: help.syntax

<PRIVATE

:: parse-help-token ( -- str/obj/f )
    ?scan-token dup search {
        { [ dup \ } eq? ] [ 2drop f ] }
        { [ dup parsing-word? ] [
            nip V{ } clone swap execute-parsing first
            dup wrapper? [ wrapped>> \ $link swap 2array ] when ] }
        [ drop ]
    } cond ;

: push-help-text ( accum sbuf obj -- accum sbuf' )
    [ dup empty? [ >string suffix! SBUF" " clone ] unless ]
    [ [ suffix! ] curry dip ] bi* ;

: push-help-space ( accum sbuf -- accum sbuf )
    {
        [ dup empty? not ]
        [ over empty? not pick ?last \ $nl eq? not and ]
    } 0|| [ CHAR: \s suffix! ] when ;

:: parse-help-text ( -- seq )
    V{ } clone SBUF" " clone [
        lexer get line>> parse-help-token [
            lexer get line>> swap - 1 > [
                \ $nl push-help-text
            ] when
        ] dip [
            [
                dup string? [
                    dup ?first ".,;:" member? [
                        [ push-help-space ] dip
                    ] unless append!
                ] [
                    [ push-help-space ]
                    [ push-help-text ] bi*
                ] if
            ] when*
        ] keep
    ] loop [ >string suffix! ] unless-empty >array ; inline

: parse-help-values ( -- seq )
    [ scan-token dup "}" = not ] [
        dup "{" = [
            drop \ } parse-until >array
        ] [
            ":" ?tail drop scan-object 2array
        ] if
    ] produce nip ;

: example-lines ( seq -- seq' )
    dup string? [ string-lines [ [ blank? ] trim ] map harvest ] when ;

: make-example ( str type -- seq )
    over string? [
        [ example-lines ] [ prefix ] bi*
    ] [ drop ] if ;

: parse-help-examples ( -- seq )
    \ } parse-until [ \ $example make-example ] { } map-as ;

: parse-help-example ( -- seq )
{'fraction_non_alphanumeric': 0.13357482454154404, 'fraction_numerical': 0.005886348200135839, 'mean_word_length': 1.5671121441022662, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4416581', 'n_tokens_mistral': 1395, 'n_tokens_neox': 1320, 'n_words': 383}