<commit_before>! Copyright (C) 2008 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: accessors assocs sequences sorting binary-search math
math.order arrays combinators kernel ;
IN: cords

<PRIVATE

TUPLE: simple-cord first second ;

M: simple-cord length
    [ first>> length ] [ second>> length ] bi + ;

M: simple-cord virtual-seq first>> ;

M: simple-cord virtual@
    2dup first>> length <
    [ first>> ] [ [ first>> length - ] [ second>> ] bi ] if ;

TUPLE: multi-cord count seqs ;

M: multi-cord length count>> ;

M: multi-cord virtual@
    dupd
    seqs>> [ first <=> ] with search nip
    [ first - ] [ second ] bi ;

M: multi-cord virtual-seq
    seqs>> [ f ] [ first second ] if-empty ;

: <cord> ( seqs -- cord )
    dup length 2 = [
        first2 simple-cord boa
    ] [
        [ 0 [ length + ] accumulate ] keep zip multi-cord boa
    ] if ;

PRIVATE>

UNION: cord simple-cord multi-cord ;

INSTANCE: cord virtual-sequence

INSTANCE: multi-cord virtual-sequence

: cord-append ( seq1 seq2 -- cord )
    {
        { [ over empty? ] [ nip ] }
        { [ dup empty? ] [ drop ] }
        { [ 2dup [ cord? ] both? ] [ [ seqs>> values ] bi@ append <cord> ] }
        { [ over cord? ] [ [ seqs>> values ] dip suffix <cord> ] }
        { [ dup cord? ] [ seqs>> values swap prefix <cord> ] }
        [ 2array <cord> ]
    } cond ;

: cord-concat ( seqs -- cord )
    {
        { [ dup empty? ] [ drop f ] }
        { [ dup length 1 = ] [ first ] }
        [
            [
                {
                    { [ dup cord? ] [ seqs>> values ] }
                    { [ dup empty? ] [ drop { } ] }
                    [ 1array ]
                } cond
            ] map concat <cord>
        ]
    } cond ;<commit_msg>give cords a shot of red bull<commit_after>! Copyright (C) 2008 <NAME>.
! See http://factorcode.org/license.txt for BSD license.
USING: accessors assocs sequences sorting binary-search math
math.order arrays combinators kernel ;
IN: cords

<PRIVATE

TUPLE: simple-cord
    { first read-only } { second read-only } ;

M: simple-cord length
    [ first>> length ] [ second>> length ] bi + ; inline

M: simple-cord virtual-seq first>> ; inline

M: simple-cord virtual@
    2dup first>> length <
    [ first>> ] [ [ first>> length - ] [ second>> ] bi ] if ; inline

TUPLE: multi-cord
    { count read-only } { seqs read-only } ;

M: multi-cord length count>> ; inline

M: multi-cord virtual@
    dupd
    seqs>> [ first <=> ] with search nip
    [ first - ] [ second ] bi ; inline

M: multi-cord virtual-seq
    seqs>> [ f ] [ first second ] if-empty ; inline

: <cord> ( seqs -- cord )
    dup length 2 = [
        first2 simple-cord boa
    ] [
        [ 0 [ length + ] accumulate ] keep zip multi-cord boa
    ] if ; inline

PRIVATE>

UNION: cord simple-cord multi-cord ;

INSTANCE: cord virtual-sequence

INSTANCE: multi-cord virtual-sequence

: cord-append ( seq1 seq2 -- cord )
    {
        { [ over empty? ] [ nip ] }
        { [ dup empty? ] [ drop ] }
        { [ 2dup [ cord? ] both? ] [ [ seqs>> values ] bi@ append <cord> ] }
        { [ over cord? ] [ [ seqs>> values ] dip suffix <cord> ] }
        { [ dup cord? ] [ seqs>> values swap prefix <cord> ] }
        [ 2array <cord> ]
    } cond ; inline

: cord-concat ( seqs -- cord )
    {
        { [ dup empty? ] [ drop f ] }
        { [ dup length 1 = ] [ first ] }
        [
            [
                {
                    { [ dup cord? ] [ seqs>> values ] }
                    { [ dup empty? ] [ drop { } ] }
                    [ 1array ]
                } cond
            ] map concat <cord>
        ]
    } cond ; inline
{'fraction_non_alphanumeric': 0.14054350809772165, 'fraction_numerical': 0.0076859730990941535, 'mean_word_length': 1.8160741885625966, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2725151', 'n_tokens_mistral': 1206, 'n_tokens_neox': 1169, 'n_words': 353}