<commit_before>'use strict'

var through = require('pull-through')

module.exports = function block (size, opts) {
  if (!opts) opts = {}
  if (typeof size === 'object') {
    opts = size
    size = opts.size
  }
  size = size || 512

  var zeroPadding

  if (opts.nopad) {
    zeroPadding = false
  } else {
    zeroPadding = typeof opts.zeroPadding !== 'undefined' ? opts.zeroPadding : true
  }

  var buffered = []
  var bufferedBytes = 0

  return through(function transform (data) {
    if (typeof data === 'number') {
      data = Buffer([data])
    }
    bufferedBytes += data.length
    buffered.push(data)

    while (bufferedBytes >= size) {
      var b = Buffer.concat(buffered)
      bufferedBytes -= size
      this.queue(b.slice(0, size))
      buffered = [ b.slice(size, b.length) ]
    }
  }, function flush (end) {
    if (bufferedBytes && zeroPadding) {
      var zeroes = new Buffer(size - bufferedBytes)
      zeroes.fill(0)
      buffered.push(zeroes)
      this.queue(Buffer.concat(buffered))
      buffered = null
    } else if (bufferedBytes) {
      this.queue(Buffer.concat(buffered))
      buffered = null
    }

    this.queue(null)
  })
}<commit_msg>fix: handle multiple flush calls<commit_after>'use strict'

var through = require('pull-through')

module.exports = function block (size, opts) {
  if (!opts) opts = {}
  if (typeof size === 'object') {
    opts = size
    size = opts.size
  }
  size = size || 512

  var zeroPadding

  if (opts.nopad) {
    zeroPadding = false
  } else {
    zeroPadding = typeof opts.zeroPadding !== 'undefined' ? opts.zeroPadding : true
  }

  var buffered = []
  var bufferedBytes = 0

  return through(function transform (data) {
    if (typeof data === 'number') {
      data = Buffer([data])
    }
    bufferedBytes += data.length
    buffered.push(data)

    while (bufferedBytes >= size) {
      var b = Buffer.concat(buffered)
      bufferedBytes -= size
      this.queue(b.slice(0, size))
      buffered = [ b.slice(size, b.length) ]
    }
  }, function flush (end) {
    if (bufferedBytes && zeroPadding) {
      var zeroes = new Buffer(size - bufferedBytes)
      zeroes.fill(0)
      buffered.push(zeroes)
      this.queue(Buffer.concat(buffered))
      buffered = null
    } else if (bufferedBytes && buffered) {
      this.queue(Buffer.concat(buffered))
      buffered = null
    }

    this.queue(null)
  })
}
{'fraction_non_alphanumeric': 0.13185433235663457, 'fraction_numerical': 0.0050230221850146506, 'mean_word_length': 2.6377473363774735, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '578475', 'n_tokens_mistral': 787, 'n_tokens_neox': 730, 'n_words': 187}