<commit_before>! Copyright (C) 2007 <NAME>
! See http://factorcode.org/license.txt for BSD license.
USING: namespaces make parser lexer kernel sequences words
quotations math accessors locals ;
IN: multiline

ERROR: bad-heredoc identifier ;

<PRIVATE

: rest-of-line ( -- seq )
    lexer get [ line-text>> ] [ column>> ] bi tail ;

: next-line-text ( -- str )
    lexer get [ next-line ] [ line-text>> ] bi ;

: (parse-here) ( -- )
    next-line-text [
        dup ";" =
        [ drop lexer get next-line ]
        [ % "\n" % (parse-here) ] if
    ] [ ";" throw-unexpected-eof ] if* ;

PRIVATE>

ERROR: text-found-before-eol string ;

: parse-here ( -- str )
    [
        rest-of-line dup [ drop ] [ text-found-before-eol ] if-empty
        (parse-here)
    ] "" make but-last ;

SYNTAX: STRING:
    scan-new-word
    parse-here 1quotation
    ( -- string ) define-inline ;

<PRIVATE

: lexer-eof? ( lexer -- ? )
    [ line>> ] [ text>> length ] bi <= ;

:: (scan-multiline-string) ( i end -- j )
    lexer get line-text>> :> text
    lexer get lexer-eof? [
        end text i start* [| j |
            i j text subseq % j end length +
        ] [
            text i short tail % CHAR: \n ,
            lexer get next-line
            0 end (scan-multiline-string)
        ] if*
    ] [ end throw-unexpected-eof ] if ;

:: (parse-multiline-string) ( end-text skip-n-chars -- str )
    [
        lexer get
        [ skip-n-chars + end-text (scan-multiline-string) ]
        change-column drop
    ] "" make ;

:: advance-same-line ( text -- )
    lexer get [ text length + ] change-column drop ;

:: (parse-til-line-begins) ( begin-text -- )
    lexer get still-parsing? [
        lexer get line-text>> begin-text sequence= [
            begin-text advance-same-line
        ] [
            lexer get line-text>> % "\n" %
            lexer get next-line
            begin-text (parse-til-line-begins)
        ] if
    ] [
        begin-text bad-heredoc
    ] if ;

: parse-til-line-begins ( begin-text -- seq )
    [ (parse-til-line-begins) ] "" make ;

PRIVATE>

: parse-multiline-string ( end-text -- str )
    1 (parse-multiline-string) ;

SYNTAX: /* "*/" parse-multiline-string drop ;

SYNTAX: HEREDOC:
    lexer get skip-blank
    rest-of-line
    lexer get next-line
    parse-til-line-begins suffix! ;

SYNTAX: DELIMITED:
    lexer get skip-blank
    rest-of-line
    lexer get next-line
    0 (parse-multiline-string) suffix! ;<commit_msg>multiline: pass the lexer around instead of using namespaces all the time.<commit_after>! Copyright (C) 2007 <NAME>
! See http://factorcode.org/license.txt for BSD license.
USING: accessors combinators kernel lexer locals make math
namespaces parser quotations sequences words ;
IN: multiline

ERROR: bad-heredoc identifier ;

<PRIVATE

: rest-of-line ( lexer -- seq )
    [ line-text>> ] [ column>> ] bi tail ;

: next-line-text ( lexer -- str )
    [ next-line ] [ line-text>> ] bi ;

: (parse-here) ( lexer -- )
    dup next-line-text [
        dup ";" =
        [ drop next-line ]
        [ % CHAR: \n , (parse-here) ] if
    ] [ drop ";" throw-unexpected-eof ] if* ;

PRIVATE>

ERROR: text-found-before-eol string ;

: parse-here ( -- str )
    [
        lexer get
        dup rest-of-line [ text-found-before-eol ] unless-empty
        (parse-here)
    ] "" make but-last ;

SYNTAX: STRING:
    scan-new-word
    parse-here 1quotation
    ( -- string ) define-inline ;

<PRIVATE

: lexer-eof? ( lexer -- ? )
    [ line>> ] [ text>> length ] bi <= ;

:: (scan-multiline-string) ( i end lexer -- j )
    lexer line-text>> :> text
    lexer lexer-eof? [
        end text i start* [| j |
            i j text subseq % j end length +
        ] [
            text i short tail % CHAR: \n ,
            lexer next-line
            0 end lexer (scan-multiline-string)
        ] if*
    ] [ end throw-unexpected-eof ] if ;

:: (parse-multiline-string) ( end-text lexer skip-n-chars -- str )
    [
        lexer
        [ skip-n-chars + end-text lexer (scan-multiline-string) ]
        change-column drop
    ] "" make ;

: advance-same-line ( lexer text -- )
    length [ + ] curry change-column drop ;

:: (parse-til-line-begins) ( begin-text lexer -- )
    lexer still-parsing? [
        lexer line-text>> begin-text sequence= [
            lexer begin-text advance-same-line
        ] [
            lexer line-text>> % CHAR: \n ,
            lexer next-line
            begin-text lexer (parse-til-line-begins)
        ] if
    ] [
        begin-text bad-heredoc
    ] if ;

: parse-til-line-begins ( begin-text lexer -- seq )
    [ (parse-til-line-begins) ] "" make ;

PRIVATE>

: parse-multiline-string ( end-text -- str )
    lexer get 1 (parse-multiline-string) ;

SYNTAX: /* "*/" parse-multiline-string drop ;

SYNTAX: HEREDOC:
    lexer get {
        [ skip-blank ]
        [ rest-of-line ]
        [ next-line ]
        [ parse-til-line-begins ]
    } cleave suffix! ;

SYNTAX: DELIMITED:
    lexer get {
        [ skip-blank ]
        [ rest-of-line ]
        [ next-line ]
        [ 0 (parse-multiline-string) ]
    } cleave suffix! ;
{'fraction_non_alphanumeric': 0.14280102141033196, 'fraction_numerical': 0.003142801021410332, 'mean_word_length': 1.914710933028048, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4801066', 'n_tokens_mistral': 1871, 'n_tokens_neox': 1758, 'n_words': 421}