<commit_before>import PnpmError from '@pnpm/error'
import * as micromatch from 'micromatch'
import path = require('path')
import execa = require('execa')
import findUp = require('find-up')
import isSubdir = require('is-subdir')

type ChangeType = 'source' | 'test'

interface ChangedDir { dir: string, changeType: ChangeType }

export default async function changedSince (packageDirs: string[], commit: string, opts: { workspaceDir: string, testPattern?: string[] }): Promise<[string[], string[]]> {
  const repoRoot = path.resolve(await findUp('.git', { cwd: opts.workspaceDir, type: 'directory' }) ?? opts.workspaceDir, '..')
  const changedDirs = (await getChangedDirsSinceCommit(commit, opts.workspaceDir, opts.testPattern ?? []))
    .map(changedDir => ({ ...changedDir, dir: path.join(repoRoot, changedDir.dir) }))
  let changedSourceDirs = changedDirs.filter(changedDir => changedDir.changeType === 'source')
  const changedPkgs: string[] = []
  for (const packageDir of packageDirs.sort((pkgDir1, pkgDir2) => pkgDir2.length - pkgDir1.length)) {
    if (
      changedSourceDirs.some(changedDir => isSubdir(packageDir, changedDir.dir))
    ) {
      changedSourceDirs = changedSourceDirs.filter((changedDir) => !isSubdir(packageDir, changedDir.dir))
      changedPkgs.push(packageDir)
    }
  }

  const ignoreDependentForPkgs = changedDirs.filter(changedDir => changedDir.changeType === 'test')
    .filter(changedDir => changedPkgs.find(pkg => changedDir.dir.startsWith(pkg)))
    .map(changedDir => changedDir.dir)

  return [changedPkgs, ignoreDependentForPkgs]
}

async function getChangedDirsSinceCommit (commit: string, workingDir: string, testPattern: string[]): Promise<ChangedDir[]> {
  let diff!: string
  try {
    diff = (
      await execa('git', [
        'diff',
        '--name-only',
        commit,
        '--',
        workingDir,
      ], { cwd: workingDir })
    ).stdout
  } catch (err) {
    throw new PnpmError('FILTER_CHANGED', `Filtering by changed packages failed. ${err.stderr as string}`)
  }
  const changedDirs = new Set<ChangedDir>()
  const dirsMatchingFilter = new Set<ChangedDir>()

  if (!diff) {
    return []
  }

  const changedFiles = diff.split('\n')

  for (const changedFile of changedFiles) {
    const dirName = path.dirname(changedFile)

    changedDirs.add({ dir: dirName, changeType: 'source' })

    if (testPattern.some(pattern => micromatch.isMatch(changedFile, pattern))) {
      dirsMatchingFilter.add({ dir: dirName, changeType: 'test' })
    }
  }

  return [...Array.from(changedDirs), ...Array.from(dirsMatchingFilter)]
}<commit_msg>fix(filter): don't match the same changed dir for several workspace projects<commit_after>import PnpmError from '@pnpm/error'
import * as micromatch from 'micromatch'
import path = require('path')
import execa = require('execa')
import findUp = require('find-up')

type ChangeType = 'source' | 'test'

interface ChangedDir { dir: string, changeType: ChangeType }

export default async function changedSince (packageDirs: string[], commit: string, opts: { workspaceDir: string, testPattern?: string[] }): Promise<[string[], string[]]> {
  const repoRoot = path.resolve(await findUp('.git', { cwd: opts.workspaceDir, type: 'directory' }) ?? opts.workspaceDir, '..')
  const changedDirs = (await getChangedDirsSinceCommit(commit, opts.workspaceDir, opts.testPattern ?? []))
    .map(changedDir => ({ ...changedDir, dir: path.join(repoRoot, changedDir.dir) }))
  const pkgChangeTypes = new Map<string, ChangeType | undefined>()
  for (const pkgDir of packageDirs) {
    pkgChangeTypes.set(pkgDir, undefined)
  }
  for (const changedDir of changedDirs) {
    let currentDir = changedDir.dir
    while (!pkgChangeTypes.has(currentDir)) {
      const nextDir = path.dirname(currentDir)
      if (nextDir === currentDir) break
      currentDir = nextDir
    }
    if (pkgChangeTypes.get(currentDir) === 'source') continue
    pkgChangeTypes.set(currentDir, changedDir.changeType)
  }

  const changedPkgs = [] as string[]
  const ignoreDependentForPkgs = [] as string[]
  for (const [changedDir, changeType] of pkgChangeTypes.entries()) {
    switch (changeType) {
    case 'source':
      changedPkgs.push(changedDir)
      break
    case 'test':
      ignoreDependentForPkgs.push(changedDir)
      break
    }
  }
  return [
    [...changedPkgs, ...ignoreDependentForPkgs],
    ignoreDependentForPkgs,
  ]
}

async function getChangedDirsSinceCommit (commit: string, workingDir: string, testPattern: string[]): Promise<ChangedDir[]> {
  let diff!: string
  try {
    diff = (
      await execa('git', [
        'diff',
        '--name-only',
        commit,
        '--',
        workingDir,
      ], { cwd: workingDir })
    ).stdout
  } catch (err) {
    throw new PnpmError('FILTER_CHANGED', `Filtering by changed packages failed. ${err.stderr as string}`)
  }
  const changedDirs = new Map<string, ChangeType>()

  if (!diff) {
    return []
  }

  const changedFiles = diff.split('\n')

  for (const changedFile of changedFiles) {
    const dir = path.dirname(changedFile)

    if (changedDirs.get(dir) === 'source') continue

    const changeType: ChangeType = testPattern.some(pattern => micromatch.isMatch(changedFile, pattern))
      ? 'test' : 'source'
    changedDirs.set(dir, changeType)
  }

  return Array.from(changedDirs.entries()).map(([dir, changeType]) => ({ dir, changeType }))
}
{'fraction_non_alphanumeric': 0.1394658753709199, 'fraction_numerical': 0.000741839762611276, 'mean_word_length': 3.9981464318813718, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '372404', 'n_tokens_mistral': 1693, 'n_tokens_neox': 1674, 'n_words': 396}