<commit_before>import * as assert from 'node:assert';
import * as cp from 'node:child_process';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import * as url from 'node:url';

import { localRepoPath } from './utils';

const NS_PER_SEC = 1e9;
const LOCAL = 'local';

// The maximum time in seconds a benchmark is allowed to run before finishing.
const maxTime = 5;
// The minimum sample size required to perform statistical analysis.
const minSamples = 5;

function runBenchmarks() {
  // Get the revisions and make things happen!
  const { benchmarks, revisions } = getArguments(process.argv.slice(2));
  const benchmarkProjects = prepareBenchmarkProjects(revisions);

  for (const benchmark of benchmarks) {
    runBenchmark(benchmark, benchmarkProjects);
  }
}

function exec(command: string, options = {}) {
  const result = cp.execSync(command, {
    encoding: 'utf-8',
    stdio: ['inherit', 'pipe', 'inherit'],
    ...options,
  });
  return result?.trimEnd();
}

interface BenchmarkProject {
  revision: string;
  projectPath: string;
}

// Build a benchmark-friendly environment for the given revision
// and returns path to its 'dist' directory.
function prepareBenchmarkProjects(
  revisionList: ReadonlyArray<string>,
): Array<BenchmarkProject> {
  const tmpDir = path.join(os.tmpdir(), 'graphql-js-benchmark');
  fs.rmSync(tmpDir, { recursive: true, force: true });
  fs.mkdirSync(tmpDir);

  const setupDir = path.join(tmpDir, 'setup');
  fs.mkdirSync(setupDir);

  return revisionList.map((revision) => {
    console.log(`🍳  Preparing ${revision}...`);
    const projectPath = path.join(setupDir, revision);
    fs.rmSync(projectPath, { recursive: true, force: true });
    fs.mkdirSync(projectPath);

    fs.cpSync(localRepoPath('benchmark'), path.join(projectPath, 'benchmark'), {
      recursive: true,
    });

    const packageJSON = {
      private: true,
      type: 'module',
      dependencies: {
        graphql: prepareNPMPackage(revision),
      },
    };
    fs.writeFileSync(
      path.join(projectPath, 'package.json'),
      JSON.stringify(packageJSON, null, 2),
    );
    exec('npm --quiet install --ignore-scripts ', { cwd: projectPath });

    return { revision, projectPath };
  });

  function prepareNPMPackage(revision: string) {
    if (revision === LOCAL) {
      const repoDir = localRepoPath();
      const archivePath = path.join(tmpDir, 'graphql-local.tgz');
      fs.renameSync(buildNPMArchive(repoDir), archivePath);
      return archivePath;
    }

    // Returns the complete git hash for a given git revision reference.
    const hash = exec(`git rev-parse "${revision}"`);

    const archivePath = path.join(tmpDir, `graphql-${hash}.tgz`);
    if (fs.existsSync(archivePath)) {
      return archivePath;
    }

    const repoDir = path.join(tmpDir, hash);
    fs.rmSync(repoDir, { recursive: true, force: true });
    fs.mkdirSync(repoDir);
    exec(`git clone --quiet "${localRepoPath()}" "${repoDir}"`);
    exec(`git checkout --quiet --detach "${hash}"`, { cwd: repoDir });
    exec('npm --quiet ci --ignore-scripts', { cwd: repoDir });
    fs.renameSync(buildNPMArchive(repoDir), archivePath);
    fs.rmSync(repoDir, { recursive: true });
    return archivePath;
  }

  function buildNPMArchive(repoDir: string) {
    exec('npm --quiet run build:npm', { cwd: repoDir });

    const distDir = path.join(repoDir, 'npmDist');
    const archiveName = exec(`npm --quiet pack ${distDir}`, { cwd: repoDir });
    return path.join(repoDir, archiveName);
  }
}

function collectSamples(modulePath: string) {
  let numOfConsequentlyRejectedSamples = 0;
  const samples = [];

  // If time permits, increase sample size to reduce the margin of error.
  const start = Date.now();
  while (samples.length < minSamples || (Date.now() - start) / 1e3 < maxTime) {
    const sample = sampleModule(modulePath);

    if (sample.involuntaryContextSwitches > 0) {<commit_msg>benchmark: use 'execOutput' from 'util.ts' (#3695)<commit_after>import * as assert from 'node:assert';
import * as cp from 'node:child_process';
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import * as url from 'node:url';

import { exec, execOutput, localRepoPath } from './utils';

const NS_PER_SEC = 1e9;
const LOCAL = 'local';

// The maximum time in seconds a benchmark is allowed to run before finishing.
const maxTime = 5;
// The minimum sample size required to perform statistical analysis.
const minSamples = 5;

function runBenchmarks() {
  // Get the revisions and make things happen!
  const { benchmarks, revisions } = getArguments(process.argv.slice(2));
  const benchmarkProjects = prepareBenchmarkProjects(revisions);

  for (const benchmark of benchmarks) {
    runBenchmark(benchmark, benchmarkProjects);
  }
}

interface BenchmarkProject {
  revision: string;
  projectPath: string;
}

// Build a benchmark-friendly environment for the given revision
// and returns path to its 'dist' directory.
function prepareBenchmarkProjects(
  revisionList: ReadonlyArray<string>,
): Array<BenchmarkProject> {
  const tmpDir = path.join(os.tmpdir(), 'graphql-js-benchmark');
  fs.rmSync(tmpDir, { recursive: true, force: true });
  fs.mkdirSync(tmpDir);

  const setupDir = path.join(tmpDir, 'setup');
  fs.mkdirSync(setupDir);

  return revisionList.map((revision) => {
    console.log(`🍳  Preparing ${revision}...`);
    const projectPath = path.join(setupDir, revision);
    fs.rmSync(projectPath, { recursive: true, force: true });
    fs.mkdirSync(projectPath);

    fs.cpSync(localRepoPath('benchmark'), path.join(projectPath, 'benchmark'), {
      recursive: true,
    });

    const packageJSON = {
      private: true,
      type: 'module',
      dependencies: {
        graphql: prepareNPMPackage(revision),
      },
    };
    fs.writeFileSync(
      path.join(projectPath, 'package.json'),
      JSON.stringify(packageJSON, null, 2),
    );
    exec('npm --quiet install --ignore-scripts ', { cwd: projectPath });

    return { revision, projectPath };
  });

  function prepareNPMPackage(revision: string) {
    if (revision === LOCAL) {
      const repoDir = localRepoPath();
      const archivePath = path.join(tmpDir, 'graphql-local.tgz');
      fs.renameSync(buildNPMArchive(repoDir), archivePath);
      return archivePath;
    }

    // Returns the complete git hash for a given git revision reference.
    const hash = execOutput(`git rev-parse "${revision}"`);

    const archivePath = path.join(tmpDir, `graphql-${hash}.tgz`);
    if (fs.existsSync(archivePath)) {
      return archivePath;
    }

    const repoDir = path.join(tmpDir, hash);
    fs.rmSync(repoDir, { recursive: true, force: true });
    fs.mkdirSync(repoDir);
    exec(`git clone --quiet "${localRepoPath()}" "${repoDir}"`);
    exec(`git checkout --quiet --detach "${hash}"`, { cwd: repoDir });
    exec('npm --quiet ci --ignore-scripts', { cwd: repoDir });
    fs.renameSync(buildNPMArchive(repoDir), archivePath);
    fs.rmSync(repoDir, { recursive: true });
    return archivePath;
  }

  function buildNPMArchive(repoDir: string) {
    exec('npm --quiet run build:npm', { cwd: repoDir });

    const distDir = path.join(repoDir, 'npmDist');
    const archiveName = execOutput(`npm --quiet pack ${distDir}`, {
      cwd: repoDir,
    });
    return path.join(repoDir, archiveName);
  }
}

function collectSamples(modulePath: string) {
  let numOfConsequentlyRejectedSamples = 0;
  const samples = [];

  // If time permits, increase sample size to reduce the margin of error.
  const start = Date.now();
  while (samples.length < minSamples || (Date.now() - start) / 1e3 < maxTime) {
    const sample = sampleModule(modulePath);

    if (sample.involuntaryContextSwitches > 0) {
{'fraction_non_alphanumeric': 0.1384753593429158, 'fraction_numerical': 0.003208418891170431, 'mean_word_length': 3.7547284929835265, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2094412', 'n_tokens_mistral': 2496, 'n_tokens_neox': 2373, 'n_words': 647}