<commit_before>'use strict'

var Transform = require('stream').Transform
var util = require('util')

exports = module.exports = function(opts) {
  return new FrameStream(opts)
}

var FrameStream = exports.FrameStream = function(opts) {
  this.opts = util._extend({
    lengthSize: 4,
    maxSize: 0,
    unbuffered: false
  }, opts)

  this.getLength = this.opts.getLength || createGetLengthMethod(this.opts.lengthSize)
  this.buffer = null
  this.frameLength = 0
  this.framePos = 0

  Transform.call(this, opts)
}

util.inherits(FrameStream, Transform)

FrameStream.prototype._transform = function transform(chunk, enc, cont) {
  while (chunk.length > 0) {
    var start = 0

    if (this.framePos === 0) {
      if (this.buffer) {
        chunk = Buffer.concat([this.buffer, chunk])
        this.buffer = null
      }

      if (chunk.length < this.opts.lengthSize) {
        this.buffer = chunk
        return cont()
      }

      this.frameLength = this.getLength(chunk)

      if (this.frameLength < 0) {
        return cont(new Error('Message length is less than zero'))
      }

      // prevent denial-of-service attacks
      if (this.opts.maxSize > 0 && this.frameLength > this.opts.maxSize) {
        return cont(new Error('Message is larger than the allowed maximum of ' + this.opts.maxSize))
      }

      start = this.opts.lengthSize
    }

    var end = start + this.frameLength - this.framePos<commit_msg>Don't name transform function as not recursing any more<commit_after>'use strict'

var Transform = require('stream').Transform
var util = require('util')

exports = module.exports = function(opts) {
  return new FrameStream(opts)
}

var FrameStream = exports.FrameStream = function(opts) {
  this.opts = util._extend({
    lengthSize: 4,
    maxSize: 0,
    unbuffered: false
  }, opts)

  this.getLength = this.opts.getLength || createGetLengthMethod(this.opts.lengthSize)
  this.buffer = null
  this.frameLength = 0
  this.framePos = 0

  Transform.call(this, opts)
}

util.inherits(FrameStream, Transform)

FrameStream.prototype._transform = function (chunk, enc, cont) {
  while (chunk.length > 0) {
    var start = 0

    if (this.framePos === 0) {
      if (this.buffer) {
        chunk = Buffer.concat([this.buffer, chunk])
        this.buffer = null
      }

      if (chunk.length < this.opts.lengthSize) {
        this.buffer = chunk
        return cont()
      }

      this.frameLength = this.getLength(chunk)

      if (this.frameLength < 0) {
        return cont(new Error('Message length is less than zero'))
      }

      // prevent denial-of-service attacks
      if (this.opts.maxSize > 0 && this.frameLength > this.opts.maxSize) {
        return cont(new Error('Message is larger than the allowed maximum of ' + this.opts.maxSize))
      }

      start = this.opts.lengthSize
    }

    var end = start + this.frameLength - this.framePos
{'fraction_non_alphanumeric': 0.1176674765706352, 'fraction_numerical': 0.006247830614370011, 'mean_word_length': 2.7971014492753623, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1488154', 'n_tokens_mistral': 909, 'n_tokens_neox': 861, 'n_words': 221}