<commit_before>import { GitProcess } from 'dugite'

type ProcessOutput = {
  output: Buffer,
  error: Buffer,
}

/**
 * Spawn a Git process and buffer the stdout and stderr streams, deferring
 * all processing work to the caller.
 *
 * @param args Array of strings to pass to the Git executable.
 * @param path The path to execute the command from.
 * @param name The name of the operation - for tracing purposes.
 * @param successExitCodes An optional array of exit codes that indicate success.
 */
export function spawnAndComplete(args: string[], path: string, name: string, successExitCodes?: Set<number>): Promise<ProcessOutput> {

  return new Promise<ProcessOutput>((resolve, reject) => {
    const commandName = `${name}: git ${args.join(' ')}`
    log.debug(`Executing ${commandName}`)

    const startTime = (performance && performance.now) ? performance.now() : null

    const process = GitProcess.spawn(args, path)
    process.stdout.setEncoding('binary')
    process.stderr.setEncoding('binary')

    const stdout = new Array<Buffer>()
    let output: Buffer | undefined
    process.stdout.on('data', (chunk) => {
      if (chunk instanceof Buffer) {
        stdout.push(chunk)
      } else {
        stdout.push(Buffer.from(chunk))
      }
    })

    const stderr = new Array<Buffer>()
    let error: Buffer | undefined
    process.stderr.on('data', (chunk) => {
      if (chunk instanceof Buffer) {
        stderr.push(chunk)
      } else {
        stderr.push(Buffer.from(chunk))
      }
    })
<commit_msg>Don't set an encoding<commit_after>import { GitProcess } from 'dugite'

type ProcessOutput = {
  output: Buffer,
  error: Buffer,
}

/**
 * Spawn a Git process and buffer the stdout and stderr streams, deferring
 * all processing work to the caller.
 *
 * @param args Array of strings to pass to the Git executable.
 * @param path The path to execute the command from.
 * @param name The name of the operation - for tracing purposes.
 * @param successExitCodes An optional array of exit codes that indicate success.
 */
export function spawnAndComplete(args: string[], path: string, name: string, successExitCodes?: Set<number>): Promise<ProcessOutput> {

  return new Promise<ProcessOutput>((resolve, reject) => {
    const commandName = `${name}: git ${args.join(' ')}`
    log.debug(`Executing ${commandName}`)

    const startTime = (performance && performance.now) ? performance.now() : null

    const process = GitProcess.spawn(args, path)

    const stdout = new Array<Buffer>()
    let output: Buffer | undefined
    process.stdout.on('data', chunk => {
      stdout.push(chunk as Buffer)
    })

    const stderr = new Array<Buffer>()
    let error: Buffer | undefined
    process.stderr.on('data', chunk => {
      stderr.push(chunk as Buffer)
    })

{'fraction_non_alphanumeric': 0.11880832735104092, 'fraction_numerical': 0.0, 'mean_word_length': 3.5097087378640777, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5361229', 'n_tokens_mistral': 792, 'n_tokens_neox': 776, 'n_words': 267}