<commit_before>'use strict'; /*jslint node: true, es5: true, indent: 2, multistr: true */
var fs = require('fs');
var test = require('tap').test;

var sv = require('..');

test('throughput test', function (t) {
  var input = [
    { index: '1', name: 'chris', time: '1:29' },
    { index: '2', name: 'daniel', time: '1:17' },
    { index: '3', name: 'lewis', time: '1:30' },
    { index: '4', name: 'stephen', time: '1:16' },
    { index: '5', name: 'larry', time: '1:31' },
  ];

  var output = [];
  var stringifier = new sv.Stringifier({peek: 2, missing: 'NA'});

  // var parser = new sv.Parser();
  var parser = new sv.Parser({encoding: stringifier.encoding, quotechar: '"'});
  parser.on('data', function(obj) {
    output.push(obj);
  });
  parser.on('end', function() {
    t.similar(output, input, 'Throughput should be transparent.');
    t.end();
  });

  stringifier.pipe(parser);
  input.forEach(function(record) {
    stringifier.write(record);
  });
  stringifier.end();
});

// var input = fs.createReadStream('test/peektest.in', {encoding: 'utf8'});
// var async = require('async');
// async.map(['peektest.in', 'peektest.out'], fs.readFile, function(err, files) {
<commit_msg>Extra line<commit_after>'use strict'; /*jslint node: true, es5: true, indent: 2, multistr: true */
var fs = require('fs');
var test = require('tap').test;

var sv = require('..');

test('throughput test', function (t) {
  var input = [
    { index: '1', name: 'chris', time: '1:29' },
    { index: '2', name: 'daniel', time: '1:17' },
    { index: '3', name: 'lewis', time: '1:30' },
    { index: '4', name: 'stephen', time: '1:16' },
    { index: '5', name: 'larry', time: '1:31' },
  ];

  var output = [];
  var stringifier = new sv.Stringifier({peek: 2, missing: 'NA'});

  var parser = new sv.Parser({encoding: stringifier.encoding, quotechar: '"'});
  parser.on('data', function(obj) {
    output.push(obj);
  });
  parser.on('end', function() {
    t.similar(output, input, 'Throughput should be transparent.');
    t.end();
  });

  stringifier.pipe(parser);
  input.forEach(function(record) {
    stringifier.write(record);
  });
  stringifier.end();
});

// var input = fs.createReadStream('test/peektest.in', {encoding: 'utf8'});
// var async = require('async');
// async.map(['peektest.in', 'peektest.out'], fs.readFile, function(err, files) {

{'fraction_non_alphanumeric': 0.2251063829787234, 'fraction_numerical': 0.020425531914893616, 'mean_word_length': 3.867494824016563, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5067476', 'n_tokens_mistral': 871, 'n_tokens_neox': 844, 'n_words': 205}