<commit_before>var pull    = require('pull-stream')
var paramap = require('pull-paramap')
var path    = require('path')
var unpack  = require('npmd-unpack').unpack
var mkdirp  = require('mkdirp')
var fs      = require('fs')
var leaves  = require('npmd-leaves')

function linkable(pkg, opts) {
  var linkRoot = (opts && opts.linkRoot)
    || path.join(process.env.HOME, '.npmd', 'linkable')
  return path.join(linkRoot, 'string' === typeof pkg ? pkg : pkg.hash)
}

//install a module with a symlink.
//adds $moduleDir/node_modules/$name -> $hash

function linkModule(moduleDir, name, hash, opts, cb) {
  if(!cb)
    cb = opts, opts = {}
  name = name.name || name
  var source = path.join(moduleDir, 'node_modules', name)
  var target = path.join(linkable(hash, opts), 'package')

  fs.readlink (source, function (err, found) {
    if(found === target)
      done()
    else if(err)
      fs.symlink(target, source, done)
    else 
      fs.unlink(source, function (err) {
        if(err) return cb(err)
        fs.symlink(target, source, done)
      })
  })

  function done (err) {
    if(err) {
      err.source = source
      err.target = target
    }
    cb(err)
  }
}

var link = 
module.exports = function (ltree, opts, cb) {
  if(!cb)
    cb = opts, opts = {}
  var dirs = {}
  var linked = {}
  pull(
    pull.values(ltree),
    paramap(function (pkg, cb) {
      var dir = linkable(pkg, opts)
      if(dirs[dir]) return cb(null, pkg)

      fs.stat(dir, function (err) {
        if(dirs[dir]) return cb(null, pkg)
        dirs[dir] = true
        if(!err) return cb(null, pkg)
        unpack(pkg, {
          cache: opts.cache,
          target: dir
        }, function (err) {
          cb(err, pkg)
        })
      })

    }),
    pull.asyncMap(function (pkg, cb) {
      //unpack to 
      //.npmd/linkable/HASH
      //then symlink to the deps
      var moduleDir = path.join(linkable(pkg), 'package')
      if(linked[pkg.hash]) return cb(null, pkg)
      linked[pkg.hash] = true
      mkdirp(path.join(moduleDir, 'node_modules'), function () {
        var n = 0
        for(var name in pkg.dependencies) {
          n ++
          linkModule(moduleDir, name, pkg.dependencies[name], next)
        }
        if(!n) cb()

        function next (err) {
          if(--n) return
          cb()
        }
      })
    }),
    pull.drain(null, function (err) {
      cb(err, ltree)
    })
  )
}

module.exports.db = function (db, config) {
  db.methods.lResolve = {type: 'async'}
  db.lResolve = function (module, opts, cb) {
    if(!cb)
      cb = opts, opts = {}

    opts.hash = true
    opts.check = false

    db.resolve(module, opts, function (err, tree) {
      cb(err, leaves(tree), tree)
    })
  }
}

module.exports.commands = function (db) {
  var start = Date.now()
  db.commands.push(function (db, config, cb) {
    var args = config._.slice()
    var cmd = args.shift()
    if('link' === cmd) {
      db.lResolve(args, config, function (err, tree, root) {
        var roots = leaves.roots(tree), n = 0
        
        for(var k in roots) {
          n ++
          ;(function (root) {
            link(tree, function (err) {
              if(err) throw err
              console.log(root.name, root.version, root.hash)
              mkdirp(path.join(process.cwd(), 'node_modules'), function () {
                linkModule(process.cwd(), root.name, root.hash, config, next)
              })
            })
          })(roots[k])
        }

        function next (err) {
          if(err) return cb(err, n = null)
          if(--n) return
          cb()
        }

      })
    } else if('lresolve' === cmd) {
      db.lResolve(args, config, function (err, tree, root) {
        console.log(JSON.stringify(tree, null, 2))
        cb()
      })
    }
    else<commit_msg>link all deps my default<commit_after>var pull    = require('pull-stream')
var paramap = require('pull-paramap')
var path    = require('path')
var unpack  = require('npmd-unpack').unpack
var mkdirp  = require('mkdirp')
var fs      = require('fs')
var leaves  = require('npmd-leaves')
var merge   = require('pull-merge')
var deps    = require('get-deps')
var pushable  = require('pull-pushable')

function linkable(pkg, opts) {
  var linkRoot = (opts && opts.linkRoot)
    || path.join(process.env.HOME, '.npmd', 'linkable')
  return path.join(linkRoot, 'string' === typeof pkg ? pkg : pkg.hash)
}

//install a module with a symlink.
//adds $moduleDir/node_modules/$name -> $hash

function linkModule(moduleDir, name, hash, opts, cb) {
  if(!cb)
    cb = opts, opts = {}
  name = name.name || name
  var source = path.join(moduleDir, 'node_modules', name)
  var target = path.join(linkable(hash, opts), 'package')

  fs.readlink (source, function (err, found) {
    console.log(source, target, found)
    if(found === target)
      done()
    else if(err)
      fs.symlink(target, source, done)
    else 
      fs.unlink(source, function (err) {
        if(err) return cb(err)
        fs.symlink(target, source, done)
      })
  })

  function done (err) {
    if(err) {
      err.source = source
      err.target = target
    }
    cb(err)
  }
}

//var spawn = require('child_process').spawn
//
//function compile(opts, cb) {
//  var cp = spawn('node-gyp', ['rebuild'], {
//    cwd: opts.target
//  })
//  cp.stdout.pipe(process.stdout)
//  cp.stderr.pipe(process.stderr)
//  cp.on('exit', function (code) {
//    cb(code === 0 ? null : new Error('exit status:'+code))
//  })
//}

function once(fun) {
  return function () {
    var args = [].slice.call(arguments)
    var cb = args.pop()
    var err = new Error('twice!')
    var i = 0
    args.push(function (err, val) {
      if(i++) throw err
      cb(err, val)
    })
    fun.apply(this, args)
  }
}


var link = 
module.exports = function (ltree, opts, cb) {
  if(!cb)
    cb = opts, opts = {}
  var dirs = {}
  var linked = {}
  var queue = pushable()

  pull(
    pull.values(ltree),
    paramap(function (pkg, cb) {
      var dir = linkable(pkg, opts)
      if(dirs[dir]) return cb(null, pkg)
      fs.stat(dir, function (err) {

        if(dirs[dir]) return cb(null, pkg)
        dirs[dir] = true
        if(!err) return cb(null, pkg)

        unpack(pkg, {
          cache: opts.cache,
          target: dir
        }, function (err) {
          console.log('unpacked', pkg.name, pkg.version)
          if(err) return cb(err)
          cb(null, pkg)
        })
      })
    }),
    pull.asyncMap(function (pkg, cb) {
      //unpack to 
      //.npmd/linkable/HASH
      //then symlink to the deps

      var moduleDir = path.join(linkable(pkg), 'package')

      if(linked[pkg.hash]) return cb(null, pkg)
      linked[pkg.hash] = true
      mkdirp(path.join(moduleDir, 'node_modules'), function () {
        var n = 0

        for(var name in pkg.dependencies) {
          n ++
          linkModule(moduleDir, name, pkg.dependencies[name], next)
        }
        if(!n) cb()

        function next (err) {
          if(--n) return
          cb()
        }
      })
    }),
    pull.drain(null, function (err) {
      cb(err, ltree)
    })
  )
}

module.exports.db = function (db, config) {
  db.methods.lResolve = {type: 'async'}
  db.lResolve = function (module, opts, cb) {
    if(!cb)
      cb = opts, opts = {}

    opts.hash = true
    opts.check = false

    db.resolve(module, opts, function (err, tree) {
      if(err) cb(err)
      cb(err, leaves(tree), tree)
    })
  }
}

var all = module.exports.all = function (tree, opts, cb) {
  var roots = leaves.roots(tree), n = 0
  opts = opts || {}
  opts.dir = opts.dir || process.cwd()

  var i = 0
  link(tree, function (err) {
    if(err) throw err
    if(i++) throw new Error('finished twice!')

    for(var k in roots) {
      n ++
      ;(function (root) {
        mkdirp(path.join(opts.dir, 'node_modules'), function () {
        console.log(root.name, root.version, root.hash)
          linkModule(opts.dir, root.name, root.hash, opts, next)
        })
      })(roots[k])
    }
  })

  function next (err) {
    if(err) return cb(err, n = null)
    if(--n) return
    cb()
  }

}

module.exports.commands = function (db) {
  var start = Date.now()
  db.commands.push(function (db, config, cb) {
    var args = config._.slice()
    var cmd = args.shift()
    if(!args.length)
      args = deps(process.cwd(), config)

    if('link' === cmd) {
      db.lResolve(args, config, function (err, tree) {
        if(err) cb(err)
        else all(tree, config, cb)
      })
    } else if('lresolve' === cmd) {
      db.lResolve(args, config, function (err, tree, root) {
        if(err) return cb(err)
        console.log(JSON.stringify(tree, null, 2))
        cb(null, tree)
      })
    }
    else
{'fraction_non_alphanumeric': 0.15872286665901, 'fraction_numerical': 0.0010336510853336397, 'mean_word_length': 2.1991182953710506, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5943609', 'n_tokens_mistral': 3042, 'n_tokens_neox': 2974, 'n_words': 725}