<commit_before>! Copyright (C) 2008, 2010 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: ascii assocs byte-arrays fry hints kernel lexer math
math.order parser sequences sequences.private strings summary
words ;
IN: tr

ERROR: bad-tr ;

M: bad-tr summary
    drop "TR: can only be used with ASCII characters" ;

<PRIVATE

: tr-nth ( n mapping -- ch ) nth-unsafe 127 bitand ; inline

: check-tr ( from to -- )
    [ [ ascii? ] all? ] both? [ bad-tr ] unless ;

: compute-tr ( quot from to -- mapping )
    [ 128 iota ] 3dip zip
    '[ [ _ call( x -- y ) _ at ] keep or ] B{ } map-as ; inline

: tr-hints ( word -- )
    { { byte-array } { string } } set-specializer ;<commit_msg>tr: zip had a string as input. call { } zip-as instead.<commit_after>! Copyright (C) 2008, 2010 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: ascii assocs byte-arrays fry hints kernel lexer math
math.order parser sequences sequences.private strings summary
words ;
IN: tr

ERROR: bad-tr ;

M: bad-tr summary
    drop "TR: can only be used with ASCII characters" ;

<PRIVATE

: tr-nth ( n mapping -- ch ) nth-unsafe 127 bitand ; inline

: check-tr ( from to -- )
    [ [ ascii? ] all? ] both? [ bad-tr ] unless ;

: compute-tr ( quot from to -- mapping )
    [ 128 iota ] 3dip { } zip-as
    '[ [ _ call( x -- y ) _ at ] keep or ] B{ } map-as ; inline

: tr-hints ( word -- )
    { { byte-array } { string } } set-specializer ;
{'fraction_non_alphanumeric': 0.13676975945017184, 'fraction_numerical': 0.020618556701030927, 'mean_word_length': 2.7142857142857144, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '305157', 'n_tokens_mistral': 544, 'n_tokens_neox': 493, 'n_words': 175}