<commit_before>import path = require('path')
import {InstalledPackage} from '.'
import mkdirp from '../fs/mkdirp'
import symlinkToModules from './symlinkToModules'
import {StorePackageMap} from '../fs/storeController'

export default function flattenDependencies (id: string, store: string, pkgs: InstalledPackage[], storePkg: StorePackageMap) {
  const newPkgs = getNewPkgs(pkgs, [id])
  const todo = newPkgs.map(newPkg => newPkg.id).concat([id])
  const tree: FlatTree = {}
  flattenPkgs(id, storePkg, 1, [], tree)
  return createFlatTree(id, store, id, storePkg, todo, tree, 1)
}

function getNewPkgs (pkgs: InstalledPackage[], keypath: string[]): InstalledPackage[] {
  return pkgs.filter(pkg => !pkg.fromCache).concat(
    pkgs
      .filter(pkg => keypath.indexOf(pkg.id) === -1)
      .reduce((newPkgs: InstalledPackage[], pkg: InstalledPackage) =>
        newPkgs.concat(getNewPkgs(pkg.dependencies, keypath.concat([pkg.id]))), []))
}

async function createFlatTree (id: string, store: string, root: string, storePkg: StorePackageMap, todo: string[], tree: FlatTree, depth: number) {
  if (todo.indexOf(id) === -1) return
  todo.splice(todo.indexOf(id), 1)

  const modules = path.join(root, 'node_modules')
  await mkdirp(modules)
  await Promise.all(
    Object.keys(tree[id])
      .map(async function (depName) {<commit_msg>fix: flat tree installation on really big packages on Windows<commit_after>import path = require('path')
import {InstalledPackage} from '.'
import mkdirp from '../fs/mkdirp'
import symlinkToModules from './symlinkToModules'
import {StorePackageMap} from '../fs/storeController'

export default function flattenDependencies (id: string, store: string, pkgs: InstalledPackage[], storePkg: StorePackageMap) {
  const newPkgs = getNewPkgs(pkgs, [id])
  const todo = new Set(newPkgs.map(newPkg => newPkg.id).concat([id]))
  const tree: FlatTree = {}
  flattenPkgs(id, storePkg, 1, [], tree)
  return createFlatTree(id, store, id, storePkg, todo, tree, 1)
}

function getNewPkgs (pkgs: InstalledPackage[], keypath: string[]): InstalledPackage[] {
  return pkgs.filter(pkg => !pkg.fromCache).concat(
    pkgs
      .filter(pkg => keypath.indexOf(pkg.id) === -1)
      .reduce((newPkgs: InstalledPackage[], pkg: InstalledPackage) =>
        newPkgs.concat(getNewPkgs(pkg.dependencies, keypath.concat([pkg.id]))), []))
}

async function createFlatTree (id: string, store: string, root: string, storePkg: StorePackageMap, todo: Set<string>, tree: FlatTree, depth: number) {
  if (!todo.has(id)) return
  todo.delete(id)

  const modules = path.join(root, 'node_modules')
  await mkdirp(modules)
  await Promise.all(
    Object.keys(tree[id])
      .map(async function (depName) {
{'fraction_non_alphanumeric': 0.1478743068391867, 'fraction_numerical': 0.0029574861367837337, 'mean_word_length': 5.013333333333334, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1061565', 'n_tokens_mistral': 933, 'n_tokens_neox': 920, 'n_words': 224}