<commit_before>! Copyright (C) 2008, 2009 <NAME>, <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: accessors assocs combinators kernel kernel.private lexer
math math.order math.parser namespaces sbufs sequences splitting
strings ;
IN: strings.parser

ERROR: bad-escape char ;

: escape ( escape -- ch )
    H{
        { CHAR: a  CHAR: \a }
        { CHAR: b  CHAR: \b }
        { CHAR: e  CHAR: \e }
        { CHAR: f  CHAR: \f }
        { CHAR: n  CHAR: \n }
        { CHAR: r  CHAR: \r }
        { CHAR: t  CHAR: \t }
        { CHAR: s  CHAR: \s }
        { CHAR: v  CHAR: \v }
        { CHAR: \s CHAR: \s }
        { CHAR: 0  CHAR: \0 }
        { CHAR: \\ CHAR: \\ }
        { CHAR: \" CHAR: \" }
    } ?at [ bad-escape ] unless ;

SYMBOL: name>char-hook

name>char-hook [
    [ "Unicode support not available" throw ]
] initialize

<PRIVATE

: hex-escape ( str -- ch str' )
    2 cut-slice [ hex> ] dip ;

: oct-escape ( str -- ch/f str' )
    dup 3 short head-slice [
        [ CHAR: 0 CHAR: 7 between? not ] find drop
    ] keep '[ _ length ] unless* [ f ] when-zero
    [ cut-slice [ oct> ] dip ] [ f swap ] if* ;

: unicode-escape ( str -- ch str' )
    "{" ?head-slice [
        CHAR: } over index cut-slice [
            dup hex> [
                nip
            ] [
                >string name>char-hook get call( name -- char )
            ] if*
        ] dip rest-slice
    ] [
        6 cut-slice [ hex> ] dip
    ] if ;
<commit_msg>strings.parser: short -> bound<commit_after>! Copyright (C) 2008, 2009 <NAME>, <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: accessors assocs combinators kernel kernel.private lexer
math math.order math.parser namespaces sbufs sequences splitting
strings ;
IN: strings.parser

ERROR: bad-escape char ;

: escape ( escape -- ch )
    H{
        { CHAR: a  CHAR: \a }
        { CHAR: b  CHAR: \b }
        { CHAR: e  CHAR: \e }
        { CHAR: f  CHAR: \f }
        { CHAR: n  CHAR: \n }
        { CHAR: r  CHAR: \r }
        { CHAR: t  CHAR: \t }
        { CHAR: s  CHAR: \s }
        { CHAR: v  CHAR: \v }
        { CHAR: \s CHAR: \s }
        { CHAR: 0  CHAR: \0 }
        { CHAR: \\ CHAR: \\ }
        { CHAR: \" CHAR: \" }
    } ?at [ bad-escape ] unless ;

SYMBOL: name>char-hook

name>char-hook [
    [ "Unicode support not available" throw ]
] initialize

<PRIVATE

: hex-escape ( str -- ch str' )
    2 cut-slice [ hex> ] dip ;

: oct-escape ( str -- ch/f str' )
    dup 3 bound head-slice [
        [ CHAR: 0 CHAR: 7 between? not ] find drop
    ] keep '[ _ length ] unless* [ f ] when-zero
    [ cut-slice [ oct> ] dip ] [ f swap ] if* ;

: unicode-escape ( str -- ch str' )
    "{" ?head-slice [
        CHAR: } over index cut-slice [
            dup hex> [
                nip
            ] [
                >string name>char-hook get call( name -- char )
            ] if*
        ] dip rest-slice
    ] [
        6 cut-slice [ hex> ] dip
    ] if ;

{'fraction_non_alphanumeric': 0.14454576156703816, 'fraction_numerical': 0.010131712259371834, 'mean_word_length': 1.5891608391608392, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7638561', 'n_tokens_mistral': 1093, 'n_tokens_neox': 1000, 'n_words': 318}