<commit_before>'use strict'
module.exports = pkg

const RegClient = require('npm-registry-client')
const tar = require('tar-fs')
const zlib = require('zlib')
const fs = require('fs')
const path = require('path')
const normalize = require('normalize-path')
const chalk = require('chalk')
const debug = require('debug')('cdn')
const streamToString = require('stream-to-string')
const buildFileTree = require('./build-file-tree')

const regClient = new RegClient()

function pkg (opts) {
  opts = opts || {}

  if (!opts.pkg) {
    throw new Error('opts.pkg is required')
  }

  if (!opts.pkg.name) {
    throw new Error('opts.pkg.name is required')
  }

  const name = opts.pkg.name

  if (!opts.pkg.version) {
    throw new Error('opts.pkg.version is required')
  }

  const version = opts.pkg.version

  if (!opts.registry) {
    throw new Error('opts.registry is required')
  }

  const registry = opts.registry

  if (!opts.storagePath) {
    throw new Error('opts.storagePath is required')
  }

  const storagePath = opts.storagePath

  const directory = getDirectory()

  const tarballURL = getTarballURL()

  const isCached = fs.existsSync(directory)

  function readJSON () {
    return require(path.resolve(directory, 'package', 'package.json'))
  }

  function download (callback) {
    if (isCached) return callback(null)

    debug('downloading tarball: ' + chalk.magenta(tarballURL))

    regClient.fetch(tarballURL, {
      auth: {
        token: registry.token,
      },
    }, (err, res) => {
      if (err) return callback(err)

      res
        .pipe(zlib.createGunzip())
        .on('error', callback)
        .pipe(tar.extract(directory))
        .on('finish', () => {
          debug('tarball downloaded: ' + chalk.magenta(tarballURL))
          buildFileTree({name, version, directory}, callback)
        })
        .on('error', callback)
    })
  }

  function streamFile (filename) {
    return new Promise((resolve, reject) => {
      const file = path.resolve(directory, 'package', filename)

      download(err => {
        if (err) {
          return reject(err)
        }

        if (!fs.existsSync(file)) {
          return reject(new Error('File not found: ' + file))
        }

        if (filename === 'package.json') {
          return resolve(fs.createReadStream(file))
        }

        const json = readJSON()

        if (json.icon && json.icon === filename) {
          return resolve(fs.createReadStream(file))
        }

        if (process.env.RESTRICTED_ACCESS) {
          return reject(new Error('I only serve package.json files and package icons these days.'))
        }
        return resolve(fs.createReadStream(file))
      })
    })
  }

  function readFile (filename) {
    return streamFile(filename).then(streamToString)
  }

  return {
    streamFile,
    readFile,
  }
<commit_msg>returning all files from a package<commit_after>'use strict'
module.exports = pkg

const RegClient = require('npm-registry-client')
const tar = require('tar-fs')
const zlib = require('zlib')
const fs = require('fs')
const path = require('path')
const normalize = require('normalize-path')
const chalk = require('chalk')
const debug = require('debug')('cdn')
const streamToString = require('stream-to-string')
const buildFileTree = require('./build-file-tree')

const regClient = new RegClient()

function pkg (opts) {
  opts = opts || {}

  if (!opts.pkg) {
    throw new Error('opts.pkg is required')
  }

  if (!opts.pkg.name) {
    throw new Error('opts.pkg.name is required')
  }

  const name = opts.pkg.name

  if (!opts.pkg.version) {
    throw new Error('opts.pkg.version is required')
  }

  const version = opts.pkg.version

  if (!opts.registry) {
    throw new Error('opts.registry is required')
  }

  const registry = opts.registry

  if (!opts.storagePath) {
    throw new Error('opts.storagePath is required')
  }

  const storagePath = opts.storagePath

  const directory = getDirectory()

  const tarballURL = getTarballURL()

  const isCached = fs.existsSync(directory)

  function download (callback) {
    if (isCached) return callback(null)

    debug('downloading tarball: ' + chalk.magenta(tarballURL))

    regClient.fetch(tarballURL, {
      auth: {
        token: registry.token,
      },
    }, (err, res) => {
      if (err) return callback(err)

      res
        .pipe(zlib.createGunzip())
        .on('error', callback)
        .pipe(tar.extract(directory))
        .on('finish', () => {
          debug('tarball downloaded: ' + chalk.magenta(tarballURL))
          buildFileTree({name, version, directory}, callback)
        })
        .on('error', callback)
    })
  }

  function streamFile (filename) {
    return new Promise((resolve, reject) => {
      const file = path.resolve(directory, 'package', filename)

      download(err => {
        if (err) {
          return reject(err)
        }

        if (!fs.existsSync(file)) {
          return reject(new Error('File not found: ' + file))
        }

        return resolve(fs.createReadStream(file))
      })
    })
  }

  function readFile (filename) {
    return streamFile(filename).then(streamToString)
  }

  return {
    streamFile,
    readFile,
  }

{'fraction_non_alphanumeric': 0.13652140377940608, 'fraction_numerical': 0.0, 'mean_word_length': 2.8083700440528636, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4107597', 'n_tokens_mistral': 1605, 'n_tokens_neox': 1538, 'n_words': 377}