<commit_before>var eos       = require('end-of-stream')
var util      = require('util')

var Readable    = require('readable-stream').Readable
var PassThrough = require('readable-stream').PassThrough
var Writable    = require('readable-stream').Writable

var headers = require('./headers')


var END_OF_TAR = new Buffer(1024)
END_OF_TAR.fill(0)

var noop = function() {}

var overflow = function(self, size) {
  size &= 511
  if (size) self.push(END_OF_TAR.slice(0, 512 - size))
}

var Sink = function(to) {
  Writable.call(this)
  this.written = 0
  this._to = to
  this._destroyed = false
}

util.inherits(Sink, Writable)

Sink.prototype._write = function(data, enc, cb) {
  this.written += data.length
  if (this._to.push(data)) return cb()
  this._to._drain = cb
}

Sink.prototype.destroy = function() {
  if (this._destroyed) return
  this._destroyed = true
  this.emit('close')
}

var Void = function() {
  Writable.call(this)
  this._destroyed = false
}

util.inherits(Void, Writable)

Void.prototype._write = function(data, enc, cb) {
  cb(new Error('No body allowed for this entry'))
}

Void.prototype.destroy = function() {
  if (this._destroyed) return
  this._destroyed = true
  this.emit('close')
}

var Pack = function(opts) {
  if (!(this instanceof Pack)) return new Pack(opts)
  Readable.call(this, opts)

  this._drain = noop
  this._finalized = false
  this._finalizing = false
  this._destroyed = false
  this._stream = null
}

util.inherits(Pack, Readable)

Pack.prototype.entry = function(header, buffer, callback) {
  if (this._stream) throw new Error('already piping an entry')
  if (this._finalized || this._destroyed) return

  if (typeof buffer === 'function') {
    callback = buffer
    buffer = null
  }

  if (!callback) callback = noop

  var self = this

  if (!header.size)  header.size = 0
  if (!header.type)  header.type = 'file'
  if (!header.mode)  header.mode = header.type === 'directory' ? 0755 : 0644
  if (!header.uid)   header.uid = 0
  if (!header.gid)   header.gid = 0
  if (!header.mtime) header.mtime = new Date()

  if (typeof buffer === 'string') buffer = new Buffer(buffer)
  if (Buffer.isBuffer(buffer)) {
    header.size = buffer.length
    this._encode(header)
    this.push(buffer)
    overflow(self, header.size)
    process.nextTick(callback)
    return new Void()
  }
  if (header.type !== 'file' && header.type !== 'contigious-file') {
    this._encode(header)
    process.nextTick(callback)
    return new Void()
  }

  var sink = new Sink(this)

  this._encode(header)<commit_msg>Use header 'mode' to detect file type if 'type' not explicitly set<commit_after>var constants = require('constants')
var eos       = require('end-of-stream')
var util      = require('util')

var Readable    = require('readable-stream').Readable
var PassThrough = require('readable-stream').PassThrough
var Writable    = require('readable-stream').Writable

var headers = require('./headers')


var END_OF_TAR = new Buffer(1024)
END_OF_TAR.fill(0)

var noop = function() {}

var overflow = function(self, size) {
  size &= 511
  if (size) self.push(END_OF_TAR.slice(0, 512 - size))
}

function mode2type(mode)
{
  switch (mode & constants.S_IFMT) {
    case constants.S_IFBLK: return 'block-device'
    case constants.S_IFCHR: return 'character-device'
    case constants.S_IFDIR: return 'directory'
    case constants.S_IFIFO: return 'fifo'
    case constants.S_IFLNK: return 'symlink'
  }

  return 'file'
}

var Sink = function(to) {
  Writable.call(this)
  this.written = 0
  this._to = to
  this._destroyed = false
}

util.inherits(Sink, Writable)

Sink.prototype._write = function(data, enc, cb) {
  this.written += data.length
  if (this._to.push(data)) return cb()
  this._to._drain = cb
}

Sink.prototype.destroy = function() {
  if (this._destroyed) return
  this._destroyed = true
  this.emit('close')
}

var Void = function() {
  Writable.call(this)
  this._destroyed = false
}

util.inherits(Void, Writable)

Void.prototype._write = function(data, enc, cb) {
  cb(new Error('No body allowed for this entry'))
}

Void.prototype.destroy = function() {
  if (this._destroyed) return
  this._destroyed = true
  this.emit('close')
}

var Pack = function(opts) {
  if (!(this instanceof Pack)) return new Pack(opts)
  Readable.call(this, opts)

  this._drain = noop
  this._finalized = false
  this._finalizing = false
  this._destroyed = false
  this._stream = null
}

util.inherits(Pack, Readable)

Pack.prototype.entry = function(header, buffer, callback) {
  if (this._stream) throw new Error('already piping an entry')
  if (this._finalized || this._destroyed) return

  if (typeof buffer === 'function') {
    callback = buffer
    buffer = null
  }

  if (!callback) callback = noop

  var self = this

  if (!header.size)  header.size = 0
  if (!header.type)  header.type = mode2type(header.mode)
  if (!header.mode)  header.mode = header.type === 'directory' ? 0755 : 0644
  if (!header.uid)   header.uid = 0
  if (!header.gid)   header.gid = 0
  if (!header.mtime) header.mtime = new Date()

  if (typeof buffer === 'string') buffer = new Buffer(buffer)
  if (Buffer.isBuffer(buffer)) {
    header.size = buffer.length
    this._encode(header)
    this.push(buffer)
    overflow(self, header.size)
    process.nextTick(callback)
    return new Void()
  }
  if (header.type !== 'file' && header.type !== 'contigious-file') {
    this._encode(header)
    process.nextTick(callback)
    return new Void()
  }

  var sink = new Sink(this)

  this._encode(header)
{'fraction_non_alphanumeric': 0.13850616388687453, 'fraction_numerical': 0.009064539521392313, 'mean_word_length': 3.6478517270429656, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '427551', 'n_tokens_mistral': 1924, 'n_tokens_neox': 1856, 'n_words': 466}