<commit_before>import fs = require('mz/fs')
import path = require('path')
import symlinkDir = require('symlink-dir')
import exists = require('path-exists')
import logger from 'pnpm-logger'
import R = require('ramda')
import pLimit = require('p-limit')
import {InstalledPackage} from '../install/installMultiple'
import {InstalledPackages, TreeNode, PackageContentInfo} from '../api/install'
import linkBins from './linkBins'
import {Package, Dependencies} from '../types'
import {Resolution} from '../resolve'
import resolvePeers, {DependencyTreeNode, DependencyTreeNodeMap} from './resolvePeers'
import logStatus from '../logging/logInstallStatus'
import updateShrinkwrap from './updateShrinkwrap'
import {shortIdToFullId} from '../fs/shrinkwrap'
import {Shrinkwrap, DependencyShrinkwrap} from 'pnpm-lockfile'
import removeOrphanPkgs from '../api/removeOrphanPkgs'
import linkIndexedDir from '../fs/linkIndexedDir'
import ncpCB = require('ncp')
import thenify = require('thenify')

const ncp = thenify(ncpCB)

export default async function (
  topPkgs: InstalledPackage[],
  rootNodeIds: string[],
  tree: {[nodeId: string]: TreeNode},
  opts: {
    force: boolean,
    global: boolean,
    baseNodeModules: string,
    bin: string,
    topParents: {name: string, version: string}[],
    shrinkwrap: Shrinkwrap,
    privateShrinkwrap: Shrinkwrap,
    production: boolean,
    optional: boolean,
    root: string,
    storePath: string,
    skipped: Set<string>,
    pkg: Package,
  }
): Promise<{
  linkedPkgsMap: DependencyTreeNodeMap,
  shrinkwrap: Shrinkwrap,
  newPkgResolvedIds: string[],
}> {
  const topPkgIds = topPkgs.map(pkg => pkg.id)
  const pkgsToLink = await resolvePeers(tree, rootNodeIds, topPkgIds, opts.topParents)
  const newShr = updateShrinkwrap(pkgsToLink, opts.shrinkwrap, opts.pkg)

  await removeOrphanPkgs(opts.privateShrinkwrap, newShr, opts.root, opts.storePath)

  let flatResolvedDeps =  R.values(pkgsToLink).filter(dep => !opts.skipped.has(dep.id))
  if (opts.production) {
    flatResolvedDeps = flatResolvedDeps.filter(dep => !dep.dev)
  }
  if (!opts.optional) {
    flatResolvedDeps = flatResolvedDeps.filter(dep => !dep.optional)
  }

  const filterOpts = {
    noDev: opts.production,
    noOptional: !opts.optional,
    skipped: opts.skipped,
  }
  const newPkgResolvedIds = await linkNewPackages(
    filterShrinkwrap(opts.privateShrinkwrap, filterOpts),
    filterShrinkwrap(newShr, filterOpts),
    pkgsToLink,
    opts
  )

  for (let pkg of flatResolvedDeps.filter(pkg => pkg.depth === 0)) {
    await symlinkDependencyTo(pkg, opts.baseNodeModules)
    logStatus({
      status: 'installed',
      pkgId: pkg.id,
    })
  }
  await linkBins(opts.baseNodeModules, opts.bin)

  return {
    linkedPkgsMap: pkgsToLink,
    shrinkwrap: newShr,
    newPkgResolvedIds,
  }
}

function filterShrinkwrap (
  shr: Shrinkwrap,
  opts: {
    noDev: boolean,
    noOptional: boolean,
    skipped: Set<string>,
  }
): Shrinkwrap {
  let pairs = R.toPairs<string, DependencyShrinkwrap>(shr.packages)
    .filter(pair => !opts.skipped.has(pair[1].id || shortIdToFullId(pair[0], shr.registry)))
  if (opts.noDev) {
    pairs = pairs.filter(pair => !pair[1].dev)
  }
  if (opts.noOptional) {
    pairs = pairs.filter(pair => !pair[1].optional)
  }
  return {
    version: shr.version,
    registry: shr.registry,
    specifiers: shr.specifiers,
    packages: R.fromPairs(pairs),
  } as Shrinkwrap
}

async function linkNewPackages (
  privateShrinkwrap: Shrinkwrap,
  shrinkwrap: Shrinkwrap,
  pkgsToLink: DependencyTreeNodeMap,
  opts: {
    force: boolean,
    global: boolean,
    baseNodeModules: string,
    optional: boolean,
  }
): Promise<string[]> {
  const nextPkgResolvedIds = R.keys(shrinkwrap.packages)
  const prevPkgResolvedIds = R.keys(privateShrinkwrap.packages)

  // TODO: what if the registries differ?
  const newPkgResolvedIds = (
      opts.force
        ? nextPkgResolvedIds
        : R.difference(nextPkgResolvedIds, prevPkgResolvedIds)
    )
    .map(shortId => shortIdToFullId(shortId, shrinkwrap.registry))

  const newPkgs = R.props<DependencyTreeNode>(newPkgResolvedIds, pkgsToLink)

  if (!opts.force) {
    // add subdependencies that have been updated
    // TODO: no need to relink everything. Can be relinked only what was changed
    for (const shortId of nextPkgResolvedIds) {
      if (privateShrinkwrap.packages[shortId] &&
        !R.equals(privateShrinkwrap.packages[shortId].dependencies, shrinkwrap.packages[shortId].dependencies)) {
        const resolvedId = shortIdToFullId(shortId, shrinkwrap.registry)
        newPkgs.push(pkgsToLink[resolvedId])
      }
    }
  }

  await linkAllModules(newPkgs, pkgsToLink, {optional: opts.optional})

  try {
    await linkAllPkgs(linkPkg, newPkgs, opts)
  } catch (err) {
    if (!err.message.startsWith('EXDEV: cross-device link not permitted')) throw err
    logger.warn(err.message)
    logger.info('Falling back to copying packages from store')
    await linkAllPkgs(copyPkg, newPkgs, opts)
  }

  await linkAllBins(newPkgs, pkgsToLink, {optional: opts.optional})

  return newPkgResolvedIds
}

const limitLinking = pLimit(16)

async function linkAllPkgs (
  linkPkg: (fetchResult: PackageContentInfo, dependency: DependencyTreeNode, opts: {
    force: boolean,
    baseNodeModules: string,<commit_msg>perf: symlink and hardlink concurrently<commit_after>import fs = require('mz/fs')
import path = require('path')
import symlinkDir = require('symlink-dir')
import exists = require('path-exists')
import logger from 'pnpm-logger'
import R = require('ramda')
import pLimit = require('p-limit')
import {InstalledPackage} from '../install/installMultiple'
import {InstalledPackages, TreeNode, PackageContentInfo} from '../api/install'
import linkBins from './linkBins'
import {Package, Dependencies} from '../types'
import {Resolution} from '../resolve'
import resolvePeers, {DependencyTreeNode, DependencyTreeNodeMap} from './resolvePeers'
import logStatus from '../logging/logInstallStatus'
import updateShrinkwrap from './updateShrinkwrap'
import {shortIdToFullId} from '../fs/shrinkwrap'
import {Shrinkwrap, DependencyShrinkwrap} from 'pnpm-lockfile'
import removeOrphanPkgs from '../api/removeOrphanPkgs'
import linkIndexedDir from '../fs/linkIndexedDir'
import ncpCB = require('ncp')
import thenify = require('thenify')

const ncp = thenify(ncpCB)

export default async function (
  topPkgs: InstalledPackage[],
  rootNodeIds: string[],
  tree: {[nodeId: string]: TreeNode},
  opts: {
    force: boolean,
    global: boolean,
    baseNodeModules: string,
    bin: string,
    topParents: {name: string, version: string}[],
    shrinkwrap: Shrinkwrap,
    privateShrinkwrap: Shrinkwrap,
    production: boolean,
    optional: boolean,
    root: string,
    storePath: string,
    skipped: Set<string>,
    pkg: Package,
  }
): Promise<{
  linkedPkgsMap: DependencyTreeNodeMap,
  shrinkwrap: Shrinkwrap,
  newPkgResolvedIds: string[],
}> {
  const topPkgIds = topPkgs.map(pkg => pkg.id)
  const pkgsToLink = await resolvePeers(tree, rootNodeIds, topPkgIds, opts.topParents)
  const newShr = updateShrinkwrap(pkgsToLink, opts.shrinkwrap, opts.pkg)

  await removeOrphanPkgs(opts.privateShrinkwrap, newShr, opts.root, opts.storePath)

  let flatResolvedDeps =  R.values(pkgsToLink).filter(dep => !opts.skipped.has(dep.id))
  if (opts.production) {
    flatResolvedDeps = flatResolvedDeps.filter(dep => !dep.dev)
  }
  if (!opts.optional) {
    flatResolvedDeps = flatResolvedDeps.filter(dep => !dep.optional)
  }

  const filterOpts = {
    noDev: opts.production,
    noOptional: !opts.optional,
    skipped: opts.skipped,
  }
  const newPkgResolvedIds = await linkNewPackages(
    filterShrinkwrap(opts.privateShrinkwrap, filterOpts),
    filterShrinkwrap(newShr, filterOpts),
    pkgsToLink,
    opts
  )

  for (let pkg of flatResolvedDeps.filter(pkg => pkg.depth === 0)) {
    await symlinkDependencyTo(pkg, opts.baseNodeModules)
    logStatus({
      status: 'installed',
      pkgId: pkg.id,
    })
  }
  await linkBins(opts.baseNodeModules, opts.bin)

  return {
    linkedPkgsMap: pkgsToLink,
    shrinkwrap: newShr,
    newPkgResolvedIds,
  }
}

function filterShrinkwrap (
  shr: Shrinkwrap,
  opts: {
    noDev: boolean,
    noOptional: boolean,
    skipped: Set<string>,
  }
): Shrinkwrap {
  let pairs = R.toPairs<string, DependencyShrinkwrap>(shr.packages)
    .filter(pair => !opts.skipped.has(pair[1].id || shortIdToFullId(pair[0], shr.registry)))
  if (opts.noDev) {
    pairs = pairs.filter(pair => !pair[1].dev)
  }
  if (opts.noOptional) {
    pairs = pairs.filter(pair => !pair[1].optional)
  }
  return {
    version: shr.version,
    registry: shr.registry,
    specifiers: shr.specifiers,
    packages: R.fromPairs(pairs),
  } as Shrinkwrap
}

async function linkNewPackages (
  privateShrinkwrap: Shrinkwrap,
  shrinkwrap: Shrinkwrap,
  pkgsToLink: DependencyTreeNodeMap,
  opts: {
    force: boolean,
    global: boolean,
    baseNodeModules: string,
    optional: boolean,
  }
): Promise<string[]> {
  const nextPkgResolvedIds = R.keys(shrinkwrap.packages)
  const prevPkgResolvedIds = R.keys(privateShrinkwrap.packages)

  // TODO: what if the registries differ?
  const newPkgResolvedIds = (
      opts.force
        ? nextPkgResolvedIds
        : R.difference(nextPkgResolvedIds, prevPkgResolvedIds)
    )
    .map(shortId => shortIdToFullId(shortId, shrinkwrap.registry))

  const newPkgs = R.props<DependencyTreeNode>(newPkgResolvedIds, pkgsToLink)

  if (!opts.force) {
    // add subdependencies that have been updated
    // TODO: no need to relink everything. Can be relinked only what was changed
    for (const shortId of nextPkgResolvedIds) {
      if (privateShrinkwrap.packages[shortId] &&
        !R.equals(privateShrinkwrap.packages[shortId].dependencies, shrinkwrap.packages[shortId].dependencies)) {
        const resolvedId = shortIdToFullId(shortId, shrinkwrap.registry)
        newPkgs.push(pkgsToLink[resolvedId])
      }
    }
  }

  await Promise.all([
    linkAllModules(newPkgs, pkgsToLink, {optional: opts.optional}),
    (async () => {
      try {
        await linkAllPkgs(linkPkg, newPkgs, opts)
      } catch (err) {
        if (!err.message.startsWith('EXDEV: cross-device link not permitted')) throw err
        logger.warn(err.message)
        logger.info('Falling back to copying packages from store')
        await linkAllPkgs(copyPkg, newPkgs, opts)
      }
    })()
  ])

  await linkAllBins(newPkgs, pkgsToLink, {optional: opts.optional})

  return newPkgResolvedIds
}

const limitLinking = pLimit(16)

async function linkAllPkgs (
  linkPkg: (fetchResult: PackageContentInfo, dependency: DependencyTreeNode, opts: {
    force: boolean,
    baseNodeModules: string,
{'fraction_non_alphanumeric': 0.11603238112961757, 'fraction_numerical': 0.0013026891225458267, 'mean_word_length': 4.127862595419847, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5478202', 'n_tokens_mistral': 3697, 'n_tokens_neox': 3648, 'n_words': 777}