<commit_before><commit_msg>just a basic test<commit_after>var test = require('tap').test;

var stream = require('stream');
var Readable = stream.Readable;
if (!Readable)
  Readable = require('readable-stream');

var Incoming = require('../incoming.js');

test('basic', function(t) {

  var s = new Readable();
  var h = { contentLength: '99', color: 'red', type: 'luftballoons' };
  var b = 'B is for\nbody. ';
  var body = b + b + b + b;
  var j = JSON.stringify(h) + '\n\n' + body;
  var chunks = [];
  for (var i = 0; i < j.length; i += 5) {
    chunks.push(new Buffer(j.slice(i, i + 5)));
  }
  chunks.push(null);

  s._read = function(n) {
    var c = chunks.shift();
    setTimeout(function() {
      s.push(c);
    });
  };

  var sp = new Incoming();
  var header;
  sp.on('header', function(h) {
    header = h;
    console.error('header', h);
  });
  sp.setEncoding('utf8');
  s.pipe(sp);

  var out = '';
  sp.on('data', function(c) {
    out += c;
  });

  var b = "B is for\nbody. "
  var expect = b + b + b + b;
  sp.on('end', function() {
    t.equal(out, expect);
    t.same(header, {
      contentLength: '99',
      color: 'red',
      type: 'luftballoons'
    });
    t.end();
  });
});
{'fraction_non_alphanumeric': 0.2049792531120332, 'fraction_numerical': 0.006639004149377593, 'mean_word_length': 2.73374613003096, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '861274', 'n_tokens_mistral': 469, 'n_tokens_neox': 459, 'n_words': 114}