<commit_before>var log = console.log
    , Brando = require( '../' )
    , Sequence = require( '../lib/filters/sequence' )
    , test = function ( mbytes, items, range ) {
        var mb = Math.max( mbytes, 1 )
            , b = new Buffer( mb * 1024 * 1024 )
            , r = range >>> 0
            , i = items >>> 0

            , seq = new Sequence( i, r )
            , stime = -1
            , etime= -1
            , k = 0
            , onFeed = function ( bytes, used_ratio ) {
                log( ' :feed, need other %d bytes, used: %d%', bytes, ( used_ratio * 100 ).toFixed( 2 ) );
            }
            , onFart = function ( result, used_ratio ) {
                etime = Date.now();
                log( '  :fart %d bytes, used: %d%', result.length, ( used_ratio * 100 ).toFixed( 2 ) );
                log( '- parsed result in: %d s\n', ( etime - stime ) / 1000 );
            }
            ;

        log( '\n- filling test buffer (%d MB)..', mb );

        for ( ; k < b.length; ++k ) b[ k ] = 0xe0 + k;

        seq.once( 'feed', onFeed );
        seq.once( 'fart', onFart );

        log( '- range: [%d, %d]..', 0, r - 1 );
        log( '- items: %d', i );
        log( '- parsing test buffer.' );

        stime = Date.now();

        seq.parse( b );

    };

test( 8, 64 * 1024, 33 );
test( 8, 64 * 1024, 32 );
test( 8, 64 * 1024, 28 );


log();<commit_msg>... On branch master modified:   bench/parse-simple-bench.js<commit_after>var log = console.log
    , Sequence = require( '../lib/filters/sequence' )
    , test = function ( mbytes, items, range ) {
        var mb = Math.max( mbytes, 1 )
            , b = new Buffer( mb * 1024 * 1024 )
            , r = range >>> 0
            , i = items >>> 0

            , seq = new Sequence( i, r )
            , stime = -1
            , etime= -1
            , k = 0
            , onFeed = function ( bytes, used_ratio ) {
                log( ' :feed, need other %d bytes, used: %d%', bytes, ( used_ratio * 100 ).toFixed( 2 ) );
            }
            , onFart = function ( result, used_ratio ) {
                etime = Date.now();
                log( '  :fart %d bytes, used: %d%', result.length, ( used_ratio * 100 ).toFixed( 2 ) );
                log( '- parsed result in: %d s\n', ( etime - stime ) / 1000 );
            }
            ;

        log( '\n- filling test buffer (%d MB)..', mb );

        for ( ; k < b.length; ++k ) b[ k ] = 0xe0 + k;

        seq.once( 'feed', onFeed );
        seq.once( 'fart', onFart );

        log( '- range: [%d, %d]..', 0, r - 1 );
        log( '- items: %d', i );
        log( '- parsing test buffer.' );

        stime = Date.now();

        seq.parse( b );

    };

test( 8, 64 * 1024, 33 );
test( 8, 64 * 1024, 32 );
test( 8, 64 * 1024, 28 );

log();
{'fraction_non_alphanumeric': 0.17602591792656588, 'fraction_numerical': 0.04103671706263499, 'mean_word_length': 1.3854077253218884, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7361671', 'n_tokens_mistral': 1042, 'n_tokens_neox': 898, 'n_words': 284}