<commit_before># 
# Utility Stream Tests
#
{
  PassThrough
}             = require 'stream'
should        = require 'should'




describe 'ReadPiper()', ->
  ReadPiper = null
  before -> {ReadPiper} = require '../../lib/streams/utils'

  it 'should preserve data order', (done) ->
    source  = new PassThrough()
    source1 = new PassThrough()
    source2 = new PassThrough()

    piper1  = new ReadPiper source1
    piper2  = new ReadPiper source2

    dest    = new PassThrough()

    # now that we have all of our streams created, hook them up
    source.pipe(piper1).pipe(piper2).pipe(dest)

    # Write to our source streams *out of order*
    source1.end 'b'
    source2.end 'c'
    source.end  'a'

    # And finally watch our writeable for data
    d = ''
    dest.on 'data', (chunk) -> d += chunk
    dest.on 'end',  ->
      d.should.equal 'abc'
      done()
<commit_msg>Improved the test a bit<commit_after># 
# Utility Stream Tests
#
{
  PassThrough
}             = require 'stream'
should        = require 'should'




describe 'ReadPiper()', ->
  ReadPiper = null
  before -> {ReadPiper} = require '../../lib/streams/utils'

  it 'should preserve data order', (done) ->
    source  = new PassThrough()
    source1 = new PassThrough()
    source2 = new PassThrough()

    piper1  = new ReadPiper source1
    piper2  = new ReadPiper source2

    dest    = new PassThrough()

    # now that we have all of our streams created, hook them up
    source.pipe(piper1).pipe(piper2).pipe(dest)

    # Write to our source streams *out of order*
    source1.write 'c'
    source.write  'a'
    source2.write 'e'
    source1.write 'd'
    source.write  'b'
    source2.write 'f'

    source.end()
    source1.end()
    source2.end()

    # And finally watch our writeable for data
    d = ''
    dest.on 'data', (chunk) -> d += chunk
    dest.on 'end',  ->
      d.should.equal 'abcdef'
      done()

{'fraction_non_alphanumeric': 0.12191276931161324, 'fraction_numerical': 0.012611665790856543, 'mean_word_length': 2.375886524822695, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1847793', 'n_tokens_mistral': 652, 'n_tokens_neox': 611, 'n_words': 191}