<commit_before>'use strict';

var Writable = require('stream').Writable;
var Parser = require('./parser.js');

function Streamer(options) {
  Writable.call(this, options);
  this.buffer = '';
  this.sep = (options && options.separator) ? options.separator : undefined;
  this.quo = (options && options.quote) ? options.quote : undefined;
}

Streamer.prototype = Object.create(
  Writable.prototype, { constructor: { value: Streamer }});


Streamer.prototype._write = function (chunk, encoding, done) {

  this.buffer = this.buffer.concat(chunk.toString());

  if (this.sep === undefined) {
    // try to detect the separator if not provided
    this.sep = require('./csv.js').detect(this.buffer);
  }


  var csv = new Parser(this.buffer, this.sep, this.quo);
  var rows = csv.File();

  if (csv.pointer > this.buffer.length) {
    this.buffer = this.buffer.slice(csv.pointer);
  }
  else {
    rows.pop();
    this.buffer = this.buffer.slice(csv.linePointer);
  }
  if (rows.length > 0) {
    var self = this;
    rows.forEach(function (row) {
        self.emit('data', row);
      }
    )
  }
  done();<commit_msg>bugfix: when a chunk finished at the end of one line<commit_after>'use strict';

var Writable = require('stream').Writable;
var Parser = require('./parser.js');

function Streamer(options) {
  Writable.call(this, options);
  this.buffer = '';
  this.sep = (options && options.separator) ? options.separator : undefined;
  this.quo = (options && options.quote) ? options.quote : undefined;
}

Streamer.prototype = Object.create(
  Writable.prototype, { constructor: { value: Streamer }});


Streamer.prototype._write = function (chunk, encoding, done) {

  this.buffer = this.buffer.concat(chunk.toString());

  if (this.sep === undefined) {
    // try to detect the separator if not provided
    this.sep = require('./csv.js').detect(this.buffer);
  }

  var csv = new Parser(this.buffer, this.sep, this.quo);
  var rows = csv.File();

  if (csv.pointer > this.buffer.length) {
    this.buffer = this.buffer.slice(csv.pointer);
  }
  else {
    if (csv.pointer === this.buffer.length) {
      rows.pop();
    }
    this.buffer = this.buffer.slice(csv.linePointer);
  }
  if (rows.length > 0) {
    var self = this;
    rows.forEach(function (row) {
        self.emit('data', row);
      }
    )
  }
  done();
{'fraction_non_alphanumeric': 0.16222030981067126, 'fraction_numerical': 0.0008605851979345956, 'mean_word_length': 3.2660550458715596, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1251722', 'n_tokens_mistral': 768, 'n_tokens_neox': 740, 'n_words': 168}