<commit_before>module.exports = deglob

var extend = require('xtend')
var findRoot = require('find-root')
var fs = require('fs')
var glob = require('glob')
var ignorePkg = require('ignore')
var os = require('os')
var parallel = require('run-parallel')
var path = require('path')
var pkgConfig = require('pkg-config')
var uniq = require('uniq')

var DEFAULT_OPTIONS = {
  useGitIgnore: true,
  usePackageJson: true,
  configKey: 'config',
  gitIgnoreFile: '.gitignore'
}

function deglob (files, opts, cb) {
  if (typeof opts === 'function') return deglob(files, null, opts)
  opts = parseOpts(opts)

  if (typeof files === 'string') files = [ files ]
  if (files.length === 0) return nextTick(cb, null, [])

  // traverse filesystem
  parallel(files.map(function (pattern) {
    return function (callback) {
      glob(pattern, {
        cwd: opts.cwd,
        ignore: opts._ignore,
        nodir: true
      }, callback)
    }
  }), function (err, results) {
    if (err) return cb(err)

    // flatten nested arrays
    var files = results.reduce(function (files, result) {
      result.forEach(function (file) {
        files.push(path.resolve(opts.cwd, file))
      })
      return files
    }, [])

    // de-dupe
    files = uniq(files)

    if (opts._gitignore) {
      files = toRelative(opts.cwd, files)
      if (os.platform() === 'win32') files = toUnix(files)
      files = opts._gitignore.filter(files)
      files = toAbsolute(opts.cwd, files)
      if (os.platform() === 'win32') files = toWin32(files)
    }
<commit_msg>update glob/deglob -- works with toolz now<commit_after>module.exports = deglob

var extend = require('../object/xtend')
var findRoot = require('../path/find-root')
var fs = require('fs')
var glob = require('./glob')
var ignorePkg = require('./ignore')
var os = require('os')
var parallel = require('../async/concurrent').parallel // run-parallel
var path = require('path')
var pkgConfig = require('../path/pkg-config')
var uniq = require('../array/uneek')
var map        = require('../array/map')

var DEFAULT_OPTIONS = {
  useGitIgnore: true,
  usePackageJson: true,
  configKey: 'config',
  gitIgnoreFile: '.gitignore'
}

function deglob (files, opts, cb) {
  if (typeof opts === 'function') return deglob(files, null, opts)
  opts = parseOpts(opts)

  if (typeof files === 'string') files = [ files ]
  if (files.length === 0) return nextTick(cb, null, [])

  // traverse filesystem
//   parallel(files, function (pattern, key, done) {
//     return glob(pattern, {
//       cwd: opts.cwd,
//       ignore: opts._ignore,
//       nodir: true
//     }, done)
//   }, function (err, results) {
//     if (err) return cb(err)

//     // flatten nested arrays
//     var files = results.reduce(function (files, result) {
//       result.forEach(function (file) {
//         files.push(path.resolve(opts.cwd, file))
//       })
//       return files
//     }, [])

//     // de-dupe
//     files = uniq(files)

//     if (opts._gitignore) {
//       files = toRelative(opts.cwd, files)
//       if (os.platform() === 'win32') files = toUnix(files)
//       files = opts._gitignore.filter(files)
//       files = toAbsolute(opts.cwd, files)
//       if (os.platform() === 'win32') files = toWin32(files)
//     }

//     return cb(null, files)
//   })
// }

  parallel(map(files, function (pattern) {
    return function (callback) {
      glob(pattern, {
        cwd: opts.cwd,
        ignore: opts._ignore,
        nodir: true
      }, function (err, paths) {
        if (err) {
          callback(err)
          return
        }
        callback(null, paths)
      })
    }
  }), function (err, results) {
    if (err) return cb(err)

    // flatten nested arrays
    var files = results.reduce(function (files, result) {
      result.forEach(function (file) {
        files.push(path.resolve(opts.cwd, file))
      })
      return files
    }, [])

    // de-dupe
    files = uniq(files)

    if (opts._gitignore) {
      files = toRelative(opts.cwd, files)
      if (os.platform() === 'win32') files = toUnix(files)
      files = opts._gitignore.filter(files)
      files = toAbsolute(opts.cwd, files)
      if (os.platform() === 'win32') files = toWin32(files)
    }

{'fraction_non_alphanumeric': 0.16306156405990016, 'fraction_numerical': 0.004753981459472308, 'mean_word_length': 2.6120171673819743, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5587357', 'n_tokens_mistral': 1383, 'n_tokens_neox': 1354, 'n_words': 335}