<commit_before>var Buffer = require('buffer').Buffer;
var EventEmitter = require('events').EventEmitter;
var sys = require('sys');


function StreamingBuffer() {
  if (!(this instanceof StreamingBuffer)) return new StreamingBuffer();

  this.buffers = [];
}
sys.inherits(StreamingBuffer, EventEmitter);
exports.StreamingBuffer = StreamingBuffer;


StreamingBuffer.prototype.push = function(buffer) {
  this.buffers.push(buffer);
  this.emit('data', buffer);
};


/**
 * Gets the next line up to CRLF.
 *
 * @return
 *   The line when there is one, undefined otherwise.
 */
StreamingBuffer.prototype.nextLine = function() {
  var bid = 0;
  var buffer = this.buffers[bid];

  while (buffer) {
    var cursor = 0;

    while (cursor < buffer.length) {
      if (buffer[cursor++] === 13 /* \r */) {
        // This potentially is a line ending. Now we only need \n as next char.
        // Make sure that we're not at the end of a buffer.
        if (cursor >= buffer.length) {
          buffer = this.buffers[++bid];
          if (!buffer) return;
          cursor = 0;
        }

        if (buffer && buffer[cursor] === 10 /* \n */) {
          // We found a line ending.
         cursor++;

          // Concat buffers if we have multiple.
          if (bid) {
            var str = '';
            for (var i = 0; i < bid; i++) {
              // Concat buffers we consumed to the end and remove them from the list.
              var b = this.buffers.shift();
              str += b.toString('ascii');
            }
            var result = str + buffer.toString('ascii', 0, cursor);
          }
          else {
            // All is in a single buffer.
            var result = buffer.toString('ascii', 0, cursor);
          }

          this.buffers[0] = buffer.slice(cursor, buffer.length);
          return result;
        }
      }
    }

    // buffer ended, get next buffer and continue with loop.
    buffer = this.buffers[++bid];
  }
};



/**
 * Gets the next line up to CRLF.
 *
 * @return
 *   The line when there is one, undefined otherwise.
 */
StreamingBuffer.prototype.nextLine2 = function() {
  var bid = 0;
  var buffer = this.buffers[bid];

  while (buffer) {
    var cursor = 0;

    while (cursor < buffer.length) {
      if (buffer[cursor++] === 13 /* \r */) {
        // This potentially is a line ending. Now we only need \n as next char.
        // Make sure that we're not at the end of a buffer.
        if (cursor >= buffer.length) {
          buffer = this.buffers[++bid];
          if (!buffer) return;
          cursor = 0;
        }

        if (buffer && buffer[cursor] === 10 /* \n */) {
          // We found a line ending.
         cursor++;

          // Concat buffers if we have multiple.
          if (bid) {
            var str = '';
            for (var i = 0; i < bid; i++) {
              // Concat buffers we consumed to the end and remove them from the list.
              var b = this.buffers.shift();
              str += b.toString('ascii');
            }
            var result = str + buffer.toString('ascii', 0, cursor);
          }
          else {
            // All is in a single buffer.
            var result = buffer.toString('ascii', 0, cursor);

          }

          this.buffers[0] = buffer.slice(cursor, buffer.length);
          return result;
        }
      }
    }

    // buffer ended, get next buffer and continue with loop.
    buffer = this.buffers[++bid];
  }
};



/**
 * Sends next N bytes to a callback function.
 *
 * @param bytes
 *   Number of bytes that should be intercepted.
 * @param callback
 *   Callback that will be called with the buffers. Arguments will be buffer and
 *   the `data` param passed to this function.
 * @param complete
 *   Callback function that will be called when the N bytes were received.
 * @param data
 *   Will be passed to callback (e.g. to identify multiple callbacks).
 * @return
 *   The line when there is one, undefined otherwise.
 */
StreamingBuffer.prototype.request = function(bytes, callback, complete, data) {
  // Don't continue processing requests for 0 bytes.
  if (!bytes) {
    complete(data);
    return true;
  }
  
  function listener(buffer) {
    if (buffer.length >= bytes) {
      // Request can be fully satisfied from this buffer.
      this.removeListener('data', listener);
      this.buffers[0] = buffer.slice(bytes, buffer.length);<commit_msg>remove stray code<commit_after>var Buffer = require('buffer').Buffer;
var EventEmitter = require('events').EventEmitter;
var sys = require('sys');


function StreamingBuffer() {
  if (!(this instanceof StreamingBuffer)) return new StreamingBuffer();

  this.buffers = [];
}
sys.inherits(StreamingBuffer, EventEmitter);
exports.StreamingBuffer = StreamingBuffer;


StreamingBuffer.prototype.push = function(buffer) {
  this.buffers.push(buffer);
  this.emit('data', buffer);
};


/**
 * Gets the next line up to CRLF.
 *
 * @return
 *   The line when there is one, undefined otherwise.
 */
StreamingBuffer.prototype.nextLine = function() {
  var bid = 0;
  var buffer = this.buffers[bid];

  while (buffer) {
    var cursor = 0;

    while (cursor < buffer.length) {
      if (buffer[cursor++] === 13 /* \r */) {
        // This potentially is a line ending. Now we only need \n as next char.
        // Make sure that we're not at the end of a buffer.
        if (cursor >= buffer.length) {
          buffer = this.buffers[++bid];
          if (!buffer) return;
          cursor = 0;
        }

        if (buffer && buffer[cursor] === 10 /* \n */) {
          // We found a line ending.
         cursor++;

          // Concat buffers if we have multiple.
          if (bid) {
            var str = '';
            for (var i = 0; i < bid; i++) {
              // Concat buffers we consumed to the end and remove them from the list.
              var b = this.buffers.shift();
              str += b.toString('ascii');
            }
            var result = str + buffer.toString('ascii', 0, cursor);
          }
          else {
            // All is in a single buffer.
            var result = buffer.toString('ascii', 0, cursor);
          }

          this.buffers[0] = buffer.slice(cursor, buffer.length);
          return result;
        }
      }
    }

    // buffer ended, get next buffer and continue with loop.
    buffer = this.buffers[++bid];
  }
};


/**
 * Sends next N bytes to a callback function.
 *
 * @param bytes
 *   Number of bytes that should be intercepted.
 * @param callback
 *   Callback that will be called with the buffers. Arguments will be buffer and
 *   the `data` param passed to this function.
 * @param complete
 *   Callback function that will be called when the N bytes were received.
 * @param data
 *   Will be passed to callback (e.g. to identify multiple callbacks).
 * @return
 *   The line when there is one, undefined otherwise.
 */
StreamingBuffer.prototype.request = function(bytes, callback, complete, data) {
  // Don't continue processing requests for 0 bytes.
  if (!bytes) {
    complete(data);
    return true;
  }
  
  function listener(buffer) {
    if (buffer.length >= bytes) {
      // Request can be fully satisfied from this buffer.
      this.removeListener('data', listener);
      this.buffers[0] = buffer.slice(bytes, buffer.length);
{'fraction_non_alphanumeric': 0.1214324917672887, 'fraction_numerical': 0.0052140504939626785, 'mean_word_length': 2.0911789652247665, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1269310', 'n_tokens_mistral': 2105, 'n_tokens_neox': 2020, 'n_words': 728}