<commit_before>import path = require('path')
import fs = require('mz/fs')
import mkdirp from '../fs/mkdirp'

export default async function hardlinkDir(existingDir: string, newDir: string) {
  await mkdirp(newDir)
  const dirs = await fs.readdir(existingDir)
  await Promise.all(
    dirs
      .map(async (relativePath: string) => {
        const existingPath = path.join(existingDir, relativePath)
        const newPath = path.join(newDir, relativePath);
        const stat = await fs.stat(existingPath)
        if (stat.isDirectory()) {
          return hardlinkDir(existingPath, newPath)
        }
        if (stat.isFile()) {
          return safeLink(existingPath, newPath)
        }
      })
  )
}

async function safeLink(existingPath: string, newPath: string) {
  try {
    await fs.link(existingPath, newPath)
  } catch (err) {
    // shouldn't normally happen, but if the file was already somehow linked,
    // the installation should not fail
    if (err.code !== 'EEXIST') throw err
  }
}<commit_msg>fix: filter out broken symlinks when linking installs<commit_after>import path = require('path')
import fs = require('mz/fs')
import mkdirp from '../fs/mkdirp'

export default async function hardlinkDir(existingDir: string, newDir: string) {
  await mkdirp(newDir)
  const dirs = await fs.readdir(existingDir)
  await Promise.all(
    dirs
      .map(async (relativePath: string) => {
        const existingPath = path.join(existingDir, relativePath)
        const newPath = path.join(newDir, relativePath)
        const stat = await fs.lstat(existingPath)
        if (stat.isSymbolicLink()) {
          // filter out broken symlinks
          if (!await fs.exists(existingPath)) {
            return;
          }
          return safeLink(existingPath, newPath)
        }
        if (stat.isDirectory()) {
          return hardlinkDir(existingPath, newPath)
        }
        if (stat.isFile()) {
          return safeLink(existingPath, newPath)
        }
      })
  )
}

async function safeLink(existingPath: string, newPath: string) {
  try {
    await fs.link(existingPath, newPath)
  } catch (err) {
    // shouldn't normally happen, but if the file was already somehow linked,
    // the installation should not fail
    if (err.code !== 'EEXIST') throw err
  }
}
{'fraction_non_alphanumeric': 0.10823838737949168, 'fraction_numerical': 0.0, 'mean_word_length': 2.792358803986711, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3475766', 'n_tokens_mistral': 674, 'n_tokens_neox': 677, 'n_words': 190}