<commit_before><commit_msg>proper transform stream to parse CSV line by line<commit_after>var stream = require('stream');
var util = require('util');

util.inherits(CSVLine, stream.Transform);

function CSVLine(options) {
  this.separator = options && options.separator ? options.separator : ';';
  stream.Transform.call(this, {
    encoding: 'utf8',
    readableObjectMode: true,
  });
}

CSVLine.prototype._transform = function(chunk, encoding, callback) {
  this.push(chunk.toString().split(this.separator));
  callback();
};

module.exports = CSVLine;
{'fraction_non_alphanumeric': 0.14594594594594595, 'fraction_numerical': 0.0018018018018018018, 'mean_word_length': 5.465116279069767, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1946411', 'n_tokens_mistral': 173, 'n_tokens_neox': 163, 'n_words': 35}