<commit_before>// commands for packing and unpacking tarballs
// this file is used by lib/cache.js

var fs = require('graceful-fs')
var path = require('path')
var writeFileAtomic = require('write-file-atomic')
var writeStreamAtomic = require('fs-write-stream-atomic')
var log = require('npmlog')
var uidNumber = require('uid-number')
var readJson = require('read-package-json')
var tar = require('tar')
var zlib = require('zlib')
var fstream = require('fstream')
var Packer = require('fstream-npm')
var iferr = require('iferr')
var inherits = require('inherits')
var npm = require('../npm.js')
var rm = require('./gently-rm.js')
var myUid = process.getuid && process.getuid()
var myGid = process.getgid && process.getgid()
var readPackageTree = require('read-package-tree')
var union = require('lodash.union')

if (process.env.SUDO_UID && myUid === 0) {
  if (!isNaN(process.env.SUDO_UID)) myUid = +process.env.SUDO_UID
  if (!isNaN(process.env.SUDO_GID)) myGid = +process.env.SUDO_GID
}

exports.pack = pack
exports.unpack = unpack

function pack (tarball, folder, pkg, cb) {
  log.verbose('tar pack', [tarball, folder])

  log.verbose('tarball', tarball)
  log.verbose('folder', folder)

  var recalculateMetadata = require('../install/deps.js').recalculateMetadata
  readPackageTree(folder, iferr(cb, function (tree) {
    recalculateMetadata(tree, log.newGroup('pack:' + pkg), iferr(cb, function () {
      pack_(tarball, folder, tree, pkg, cb)
    }))
  }))
}

function BundledPacker (props) {
  Packer.call(this, props)
}
inherits(BundledPacker, Packer)

BundledPacker.prototype.applyIgnores = function (entry, partial, entryObj) {
  // package.json files can never be ignored.
  if (entry === 'package.json') return true

  // readme files should never be ignored.
  if (entry.match(/^readme(\.[^\.]*)$/i)) return true

  // license files should never be ignored.
  if (entry.match(/^(license|licence)(\.[^\.]*)?$/i)) return true

  // changelogs should never be ignored.
  if (entry.match(/^(changes|changelog|history)(\.[^\.]*)?$/i)) return true

  // special rules.  see below.
  if (entry === 'node_modules' && this.packageRoot) return true

  // some files are *never* allowed under any circumstances
  if (entry === '.git' ||
      entry === '.lock-wscript' ||
      entry.match(/^\.wafpickle-[0-9]+$/) ||
      entry === 'CVS' ||
      entry === '.svn' ||
      entry === '.hg' ||
      entry.match(/^\..*\.swp$/) ||
      entry === '.DS_Store' ||
      entry.match(/^\._/) ||
      entry === 'npm-debug.log'
    ) {
    return false
  }

  // in a node_modules folder, we only include bundled dependencies
  // also, prevent packages in node_modules from being affected
  // by rules set in the containing package, so that
  // bundles don't get busted.
  // Also, once in a bundle, everything is installed as-is
  // To prevent infinite cycles in the case of cyclic deps that are
  // linked with npm link, even in a bundle, deps are only bundled
  // if they're not already present at a higher level.
  if (this.bundleMagic) {
    // bubbling up.  stop here and allow anything the bundled pkg allows
    if (entry.indexOf('/') !== -1) return true

    // never include the .bin.  It's typically full of platform-specific
    // stuff like symlinks and .cmd files anyway.
    if (entry === '.bin') return false

    // the package root.
    var p = this.parent
    // the package before this one.
    var pp = p && p.parent

    // if this entry has already been bundled, and is a symlink,
    // and it is the *same* symlink as this one, then exclude it.
    if (pp && pp.bundleLinks && this.bundleLinks &&
        pp.bundleLinks[entry] &&
        pp.bundleLinks[entry] === this.bundleLinks[entry]) {
      return false
    }

    // since it's *not* a symbolic link, if we're *already* in a bundle,
    // then we should include everything.
    if (pp && pp.package && pp.basename === 'node_modules') {
      return true
    }

    // only include it at this point if it's a bundleDependency
    return this.isBundled(entry)
  }
  // if (this.bundled) return true

  return Packer.prototype.applyIgnores.call(this, entry, partial, entryObj)
}

function nameMatch (name) { return function (other) { return name === other.package.name } }

function pack_ (tarball, folder, tree, pkg, cb) {
  var flattenTree = require('../install/flatten-tree.js')
  var flatTree = flattenTree(tree)
  function InstancePacker (props) {
    BundledPacker.call(this, props)
  }
  inherits(InstancePacker, BundledPacker)
  InstancePacker.prototype.isBundled = function (name) {
    var bd = this.package && this.package.bundleDependencies
    if (!bd) return false

    if (!Array.isArray(bd)) {
      throw new Error(this.package.name + '\'s `bundledDependencies` should ' +
                      'be an array')
    }

    if (bd.indexOf(name) !== -1) return true
    var pkg = tree.children.filter(nameMatch(name))[0]
    if (!pkg) return false
    var requiredBy = union([], pkg.package._requiredBy)
    var seen = {}
    while (requiredBy.length) {
      var req = requiredBy.shift()
      if (seen[req]) continue<commit_msg>pack: Don't bother reading the tree if nothing is bundled<commit_after>// commands for packing and unpacking tarballs
// this file is used by lib/cache.js

var fs = require('graceful-fs')
var path = require('path')
var writeFileAtomic = require('write-file-atomic')
var writeStreamAtomic = require('fs-write-stream-atomic')
var log = require('npmlog')
var uidNumber = require('uid-number')
var readJson = require('read-package-json')
var tar = require('tar')
var zlib = require('zlib')
var fstream = require('fstream')
var Packer = require('fstream-npm')
var iferr = require('iferr')
var inherits = require('inherits')
var npm = require('../npm.js')
var rm = require('./gently-rm.js')
var myUid = process.getuid && process.getuid()
var myGid = process.getgid && process.getgid()
var readPackageTree = require('read-package-tree')
var union = require('lodash.union')
var flattenTree = require('../install/flatten-tree.js')

if (process.env.SUDO_UID && myUid === 0) {
  if (!isNaN(process.env.SUDO_UID)) myUid = +process.env.SUDO_UID
  if (!isNaN(process.env.SUDO_GID)) myGid = +process.env.SUDO_GID
}

exports.pack = pack
exports.unpack = unpack

function pack (tarball, folder, pkg, cb) {
  log.verbose('tar pack', [tarball, folder])

  log.verbose('tarball', tarball)
  log.verbose('folder', folder)

  readJson(path.join(folder, 'package.json'), function (er, pkg) {
    if (er || !pkg.bundleDependencies) {
      pack_(tarball, folder, null, null, pkg, cb)
    } else {
      // we require this at runtime due to load-order issues, because recursive
      // requires fail if you replace the exports object, and we do, not in deps, but
      // in a dep of it.
      var recalculateMetadata = require('../install/deps.js').recalculateMetadata

      readPackageTree(folder, iferr(cb, function (tree) {
        recalculateMetadata(tree, log.newGroup('pack:' + pkg), iferr(cb, function () {
          pack_(tarball, folder, tree, flattenTree(tree), pkg, cb)
        }))
      }))
    }
  })
}

function BundledPacker (props) {
  Packer.call(this, props)
}
inherits(BundledPacker, Packer)

BundledPacker.prototype.applyIgnores = function (entry, partial, entryObj) {
  // package.json files can never be ignored.
  if (entry === 'package.json') return true

  // readme files should never be ignored.
  if (entry.match(/^readme(\.[^\.]*)$/i)) return true

  // license files should never be ignored.
  if (entry.match(/^(license|licence)(\.[^\.]*)?$/i)) return true

  // changelogs should never be ignored.
  if (entry.match(/^(changes|changelog|history)(\.[^\.]*)?$/i)) return true

  // special rules.  see below.
  if (entry === 'node_modules' && this.packageRoot) return true

  // some files are *never* allowed under any circumstances
  if (entry === '.git' ||
      entry === '.lock-wscript' ||
      entry.match(/^\.wafpickle-[0-9]+$/) ||
      entry === 'CVS' ||
      entry === '.svn' ||
      entry === '.hg' ||
      entry.match(/^\..*\.swp$/) ||
      entry === '.DS_Store' ||
      entry.match(/^\._/) ||
      entry === 'npm-debug.log'
    ) {
    return false
  }

  // in a node_modules folder, we only include bundled dependencies
  // also, prevent packages in node_modules from being affected
  // by rules set in the containing package, so that
  // bundles don't get busted.
  // Also, once in a bundle, everything is installed as-is
  // To prevent infinite cycles in the case of cyclic deps that are
  // linked with npm link, even in a bundle, deps are only bundled
  // if they're not already present at a higher level.
  if (this.bundleMagic) {
    // bubbling up.  stop here and allow anything the bundled pkg allows
    if (entry.indexOf('/') !== -1) return true

    // never include the .bin.  It's typically full of platform-specific
    // stuff like symlinks and .cmd files anyway.
    if (entry === '.bin') return false

    // the package root.
    var p = this.parent
    // the package before this one.
    var pp = p && p.parent

    // if this entry has already been bundled, and is a symlink,
    // and it is the *same* symlink as this one, then exclude it.
    if (pp && pp.bundleLinks && this.bundleLinks &&
        pp.bundleLinks[entry] &&
        pp.bundleLinks[entry] === this.bundleLinks[entry]) {
      return false
    }

    // since it's *not* a symbolic link, if we're *already* in a bundle,
    // then we should include everything.
    if (pp && pp.package && pp.basename === 'node_modules') {
      return true
    }

    // only include it at this point if it's a bundleDependency
    return this.isBundled(entry)
  }
  // if (this.bundled) return true

  return Packer.prototype.applyIgnores.call(this, entry, partial, entryObj)
}

function nameMatch (name) { return function (other) { return name === other.package.name } }

function pack_ (tarball, folder, tree, flatTree, pkg, cb) {
  function InstancePacker (props) {
    BundledPacker.call(this, props)
  }
  inherits(InstancePacker, BundledPacker)
  InstancePacker.prototype.isBundled = function (name) {
    var bd = this.package && this.package.bundleDependencies
    if (!bd) return false

    if (!Array.isArray(bd)) {
      throw new Error(this.package.name + '\'s `bundledDependencies` should ' +
                      'be an array')
    }
    if (!tree) return false

    if (bd.indexOf(name) !== -1) return true
    var pkg = tree.children.filter(nameMatch(name))[0]
    if (!pkg) return false
    var requiredBy = union([], pkg.package._requiredBy)
    var seen = {}
    while (requiredBy.length) {
      var req = requiredBy.shift()
      if (seen[req]) continue
{'fraction_non_alphanumeric': 0.14488424197162061, 'fraction_numerical': 0.0011202389843166542, 'mean_word_length': 3.1555469356089993, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3344230', 'n_tokens_mistral': 3459, 'n_tokens_neox': 3325, 'n_words': 1035}