<commit_before>
/*
new technique for tarring

goal:
1. get rid of the -C and --strip-components nonsense
2. always have a consistent folder name in the package.tgz files

adding a folder:
1. tar into tmp/random/package.tgz
2. untar into tmp/random/contents/{blah}
3. rename {blah} to "package"
4. tar tmp/random/package to cache/n/v/package.tgz
5. untar cache/n/v/package.tgz into cache/n/v/package
6. rm tmp/random and tmp/random.tgz

Adding a url:
1. fetch to tmp/random.tgz
2. goto folder(2)

adding a name/version:
1. registry.get(name, version)
2. if response isn't 304, add url(dist.tarball)

adding a local tarball:
1. untar to tmp/random/{blah}
2. goto folder(2)
*/

exports = module.exports = cache
exports.read = read
exports.clean = clean
exports.unpack = unpack

var mkdir = require("./utils/mkdir-p")
  , exec = require("./utils/exec")
  , pipe = exec.pipe
  , spawn = exec.spawn
  , fetch = require("./utils/fetch")
  , npm = require("../npm")
  , fs = require("./utils/graceful-fs")
  , rm = require("./utils/rm-rf")
  , readJson = require("./utils/read-json")
  , registry = require("./utils/registry")
  , log = require("./utils/log")
  , path = require("path")
  , sys = require("./utils/sys")
  , output = require("./utils/output")
  , sha = require("./utils/sha")

cache.usage = "npm cache add <tarball file>"
            + "\nnpm cache add <folder>"
            + "\nnpm cache add <tarball url>"
            + "\nnpm cache add <name>@<version>"
            + "\nnpm cache ls [<path>]"
            + "\nnpm cache clean [<pkg>[@<version>]]"

cache.completion = function(args, index, cb) {
  var remotePkgs = require("./utils/completion/remote-packages")
    , getCompletions = require("./utils/completion/get-completions")
    , subcmdList = ["add", "clean", "ls"]
    , subcmd = args[0] || ""

  if (subcmdList.indexOf(subcmd) !== -1) {
    if (subcmd === "add") {
      remotePkgs(args.slice(1), index - 1, true, false, false, cb)
    } else if (subcmd === "clean") {
      ls_("", npm.cache, function (er, cached) {
        if (er) return cb(er)
        cb(null, getCompletions(args[1] || "", cached))
      })
    }
  } else if (index < 3) cb(null, getCompletions(subcmd, subcmdList))
}

function cache (args, cb) {
  var cmd = args.shift()
  switch (cmd) {
    case "clean": return clean(args, cb)
    case "ls": return ls(args, cb)
    case "add": return add(args, cb)
    default: return cb(new Error(
      "Invalid cache action: "+cmd))
  }
}

// if the pkg and ver are in the cache, then
// just do a readJson and return.
// if they're not, then fetch them from the registry.
var cacheSeen = {}
function read (name, ver, cb) {
  var jsonFile = path.join(npm.cache, name, ver, "package.json")
  function c (er, data) {
    if (!er) cacheSeen[data._id] = data
    if (data) deprCheck(data)
    return cb(er, data)
  }
  if (name+"@"+ver in cacheSeen) {
    return cb(null, cacheSeen[name+"@"+ver])
  }
  readJson(jsonFile, function (er, data) {
    if (er) return addNameVersion(name, ver, c)
    deprCheck(data)
    c(er, data)
  })
}

// npm cache ls [<pkg>[@<ver>]]
function ls (args, cb) {
  if (args[0] && args[0].indexOf("@") !== -1) {
    args = args[0].split("@")
  }
  var show = path.join.apply(path, args)
    , read = npm.cache
    , msg = "cache ls"+(show?" "+show:"")
  read = path.join(read, show)
  ls_(show, read, function(er, files) {
    var prefix = (show ? show + "/" : "")<commit_msg>sometimes directories are empty, or not found<commit_after>
/*
new technique for tarring

goal:
1. get rid of the -C and --strip-components nonsense
2. always have a consistent folder name in the package.tgz files

adding a folder:
1. tar into tmp/random/package.tgz
2. untar into tmp/random/contents/{blah}
3. rename {blah} to "package"
4. tar tmp/random/package to cache/n/v/package.tgz
5. untar cache/n/v/package.tgz into cache/n/v/package
6. rm tmp/random and tmp/random.tgz

Adding a url:
1. fetch to tmp/random.tgz
2. goto folder(2)

adding a name/version:
1. registry.get(name, version)
2. if response isn't 304, add url(dist.tarball)

adding a local tarball:
1. untar to tmp/random/{blah}
2. goto folder(2)
*/

exports = module.exports = cache
exports.read = read
exports.clean = clean
exports.unpack = unpack

var mkdir = require("./utils/mkdir-p")
  , exec = require("./utils/exec")
  , pipe = exec.pipe
  , spawn = exec.spawn
  , fetch = require("./utils/fetch")
  , npm = require("../npm")
  , fs = require("./utils/graceful-fs")
  , rm = require("./utils/rm-rf")
  , readJson = require("./utils/read-json")
  , registry = require("./utils/registry")
  , log = require("./utils/log")
  , path = require("path")
  , sys = require("./utils/sys")
  , output = require("./utils/output")
  , sha = require("./utils/sha")

cache.usage = "npm cache add <tarball file>"
            + "\nnpm cache add <folder>"
            + "\nnpm cache add <tarball url>"
            + "\nnpm cache add <name>@<version>"
            + "\nnpm cache ls [<path>]"
            + "\nnpm cache clean [<pkg>[@<version>]]"

cache.completion = function(args, index, cb) {
  var remotePkgs = require("./utils/completion/remote-packages")
    , getCompletions = require("./utils/completion/get-completions")
    , subcmdList = ["add", "clean", "ls"]
    , subcmd = args[0] || ""

  if (subcmdList.indexOf(subcmd) !== -1) {
    if (subcmd === "add") {
      remotePkgs(args.slice(1), index - 1, true, false, false, cb)
    } else if (subcmd === "clean") {
      ls_("", npm.cache, function (er, cached) {
        if (er) return cb(er)
        cb(null, getCompletions(args[1] || "", cached))
      })
    }
  } else if (index < 3) cb(null, getCompletions(subcmd, subcmdList))
}

function cache (args, cb) {
  var cmd = args.shift()
  switch (cmd) {
    case "clean": return clean(args, cb)
    case "ls": return ls(args, cb)
    case "add": return add(args, cb)
    default: return cb(new Error(
      "Invalid cache action: "+cmd))
  }
}

// if the pkg and ver are in the cache, then
// just do a readJson and return.
// if they're not, then fetch them from the registry.
var cacheSeen = {}
function read (name, ver, cb) {
  var jsonFile = path.join(npm.cache, name, ver, "package.json")
  function c (er, data) {
    if (!er) cacheSeen[data._id] = data
    if (data) deprCheck(data)
    return cb(er, data)
  }
  if (name+"@"+ver in cacheSeen) {
    return cb(null, cacheSeen[name+"@"+ver])
  }
  readJson(jsonFile, function (er, data) {
    if (er) return addNameVersion(name, ver, c)
    deprCheck(data)
    c(er, data)
  })
}

// npm cache ls [<pkg>[@<ver>]]
function ls (args, cb) {
  if (args[0] && args[0].indexOf("@") !== -1) {
    args = args[0].split("@")
  }
  var show = path.join.apply(path, args)
    , read = npm.cache
    , msg = "cache ls"+(show?" "+show:"")
  read = path.join(read, show)
  ls_(show, read, function(er, files) {
    files = files || []
    var prefix = (show ? show + "/" : "")
{'fraction_non_alphanumeric': 0.1750939034960994, 'fraction_numerical': 0.008379081190407396, 'mean_word_length': 2.9971131639722866, 'pattern_counts': {'":': 6, '<': 25, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3975729', 'n_tokens_mistral': 2515, 'n_tokens_neox': 2428, 'n_words': 704}