<commit_before>
/**
 * Module dependencies.
 */

var assert = require('assert');
var debug = require('debug')('stream-parser');

/**
 * Module exports.
 */

module.exports = Parser;

/**
 * The `Parser` stream mixin works with either `Writable` or `Transform` stream
 * instances/subclasses. Provides a convenient generic "parsing" API:
 *
 *   _bytes(n, cb) - buffers "n" bytes and then calls "cb" with the "chunk"
 *
 * If you extend a `Transform` stream, then the `_passthrough()` function is also
 * added:
 *
 *   _passthrough(n, cb) - passes through "n" bytes untouched and then calls "cb"
 *
 * @param {Stream} stream Transform or Writable stream instance to extend
 * @api public
 */

function Parser (stream) {
  var isTransform = stream && 'function' == typeof stream._transform;
  var isWritable = stream && 'function' == typeof stream._write;

  if (!isTransform && !isWritable) throw new Error('must pass a Writable or Transform stream in');
  debug('extending Parser into stream');

  // both Transform streams and Writable streams get `_bytes()`
  stream._bytes = _bytes;

  // only Transform streams get the `_passthrough()` function
  if (isTransform) stream._passthrough = _passthrough;

  // take control of the streams2 callback functions for this stream
  if (isTransform) {
    stream._transform = transform;
  } else {
    stream._write = write;
  }
}

function init (stream) {
  debug('initializing parser stream');

  // number of bytes left to parser for the next "chunk"
  stream._parserBytesLeft = 0;

  // array of Buffer instances that make up the next "chunk"
  stream._parserBuffers = [];

  // number of bytes parsed so far for the next "chunk"
  stream._parserBuffered = 0;

  // flag the keeps track of if we're buffering or passing-through
  stream._parserBuffering = false;

  // the callback for the next "chunk"
  stream._parserCallback = null;

  stream._parserInit = true;
}

/**
 * Buffers `n` bytes and then invokes `fn` once that amount has been collected.
 *
 * @param {Number} n the number of bytes to buffer
 * @param {Function} fn callback function to invoke when `n` bytes are buffered
 * @api public
 */

function _bytes (n, fn) {
  assert(!this._parserCallback, 'there is already a "callback" set!');
  assert(isFinite(n) && n > 0, 'can only buffer a finite number of bytes > 0, got "' + n + '"');
  if (!this._parserInit) init(this);
  debug('buffering "%d" bytes', n);
  this._parserBytesLeft = n;
  this._parserCallback = fn;
  this._parserBuffering = true;
}

/**
 * Passes through `n` bytes to the readable side of this stream untouched,
 * then invokes `fn` once that amount has been passed through.
 *
 * @param {Number} n the number of bytes to pass through
 * @param {Function} fn callback function to invoke when `n` bytes have passed through
 * @api public
 */

function _passthrough (n, fn) {
  assert(!this._parserCallback, 'There is already a "callback" set!');
  assert(n > 0, 'can only pass through > 0 bytes, got "' + n + '"');
  if (!this._parserInit) init(this);
  debug('passing through "%d" bytes', n);
  this._parserBytesLeft = n;
  this._parserCallback = fn;
  this._parserBuffering = false;
}

/**
 * The `_write()` callback function implementation.
 *
 * @api private
 */

function write (chunk, fn) {
  if (!this._parserInit) init(this);
  debug('write(%d bytes)', chunk.length);
  data(this, chunk, null, fn);
}

/**
 * The `_transform()` callback function implementation.
 *
 * @api private
 */


function transform (chunk, output, fn) {
  if (!this._parserInit) init(this);
  debug('transform(%d bytes)', chunk.length);
  data(this, chunk, output, fn);
}

/**
 * The internal buffering/passthrough logic...<commit_msg>parser: updates for node v0.9.12 streams2 API changes<commit_after>
/**
 * Module dependencies.
 */

var assert = require('assert');
var debug = require('debug')('stream-parser');

/**
 * Module exports.
 */

module.exports = Parser;

/**
 * The `Parser` stream mixin works with either `Writable` or `Transform` stream
 * instances/subclasses. Provides a convenient generic "parsing" API:
 *
 *   _bytes(n, cb) - buffers "n" bytes and then calls "cb" with the "chunk"
 *
 * If you extend a `Transform` stream, then the `_passthrough()` function is also
 * added:
 *
 *   _passthrough(n, cb) - passes through "n" bytes untouched and then calls "cb"
 *
 * @param {Stream} stream Transform or Writable stream instance to extend
 * @api public
 */

function Parser (stream) {
  var isTransform = stream && 'function' == typeof stream._transform;
  var isWritable = stream && 'function' == typeof stream._write;

  if (!isTransform && !isWritable) throw new Error('must pass a Writable or Transform stream in');
  debug('extending Parser into stream');

  // both Transform streams and Writable streams get `_bytes()`
  stream._bytes = _bytes;

  // only Transform streams get the `_passthrough()` function
  if (isTransform) stream._passthrough = _passthrough;

  // take control of the streams2 callback functions for this stream
  if (isTransform) {
    stream._transform = transform;
  } else {
    stream._write = write;
  }
}

function init (stream) {
  debug('initializing parser stream');

  // number of bytes left to parser for the next "chunk"
  stream._parserBytesLeft = 0;

  // array of Buffer instances that make up the next "chunk"
  stream._parserBuffers = [];

  // number of bytes parsed so far for the next "chunk"
  stream._parserBuffered = 0;

  // flag the keeps track of if we're buffering or passing-through
  stream._parserBuffering = false;

  // the callback for the next "chunk"
  stream._parserCallback = null;

  // XXX: backwards compat with the old Transform API... remove at some point..
  if ('function' == typeof stream.push) {
    stream._parserOutput = stream.push.bind(stream);
  }

  stream._parserInit = true;
}

/**
 * Buffers `n` bytes and then invokes `fn` once that amount has been collected.
 *
 * @param {Number} n the number of bytes to buffer
 * @param {Function} fn callback function to invoke when `n` bytes are buffered
 * @api public
 */

function _bytes (n, fn) {
  assert(!this._parserCallback, 'there is already a "callback" set!');
  assert(isFinite(n) && n > 0, 'can only buffer a finite number of bytes > 0, got "' + n + '"');
  if (!this._parserInit) init(this);
  debug('buffering "%d" bytes', n);
  this._parserBytesLeft = n;
  this._parserCallback = fn;
  this._parserBuffering = true;
}

/**
 * Passes through `n` bytes to the readable side of this stream untouched,
 * then invokes `fn` once that amount has been passed through.
 *
 * @param {Number} n the number of bytes to pass through
 * @param {Function} fn callback function to invoke when `n` bytes have passed through
 * @api public
 */

function _passthrough (n, fn) {
  assert(!this._parserCallback, 'There is already a "callback" set!');
  assert(n > 0, 'can only pass through > 0 bytes, got "' + n + '"');
  if (!this._parserInit) init(this);
  debug('passing through "%d" bytes', n);
  this._parserBytesLeft = n;
  this._parserCallback = fn;
  this._parserBuffering = false;
}

/**
 * The `_write()` callback function implementation.
 *
 * @api private
 */

function write (chunk, encoding, fn) {
  if (!this._parserInit) init(this);
  debug('write(%d bytes)', chunk.length);

  // XXX: old Writable stream API compat... remove at some point...
  if ('function' == typeof encoding) fn = encoding;

  data(this, chunk, null, fn);
}

/**
 * The `_transform()` callback function implementation.
 *
 * @api private
 */


function transform (chunk, output, fn) {
  if (!this._parserInit) init(this);
  debug('transform(%d bytes)', chunk.length);

  // XXX: old Transform stream API compat... remove at some point...
  if ('function' != typeof output) {
    output = this._parserOutput;
  }

  data(this, chunk, output, fn);
}

/**
 * The internal buffering/passthrough logic...
{'fraction_non_alphanumeric': 0.12859495755732928, 'fraction_numerical': 0.0024071962498416317, 'mean_word_length': 3.746843054720385, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1621604', 'n_tokens_mistral': 2410, 'n_tokens_neox': 2335, 'n_words': 887}