<commit_before>const fs = require('fs');
const path = require('path');
const childProcess = require('child_process');
const del = require('node-delete');
const Zip = require('node-zip');
const colors = require('colors');
const diff = require('diff');

module.exports = function(args, resolve) {
  let outputData = {
    removedFiles: [],
    addedFiles: [],
    changedFiles: {}
  };

  // get manufacturer and fixture data later used by export plugins
  const fixtureDir = path.join(__dirname, '..', 'fixtures');
  const manufacturers = JSON.parse(fs.readFileSync(
    path.join(fixtureDir, 'manufacturers.json')
  ));
  let library = [];
  let fixtures = [];
  for (let fixture of args.fixtures) {
    // fixture path is relative to user's working directory
    const fixPath = path.join(process.env.PWD, fixture);
    library.push({
      manufacturerKey: path.basename(path.dirname(fixPath)),
      fixtureKey: path.basename(fixPath, path.extname(fixPath))
    });
    fixtures.push(path.join('fixtures', path.relative(fixtureDir, fixPath)));
  }

  console.log();
  console.log('## Diffing plugin output');
  console.log(`# plugin: ${args.plugin}`);
  console.log(`# ref: ${args.ref}`);
  console.log(`# fixtures: ${fixtures}`);
  console.log();

  // delete old temp folder and recreate it
  const tempDir = path.join(__dirname, '..', 'tmp');
  if (process.env.PWD.match(tempDir)) {
    console.error(colors.red('[Error]') + ' This script can\'t be run from inside the tmp directory.');
    process.exit(1);
  }
  del.sync([tempDir], {force: true});
  fs.mkdirSync(tempDir);
  const currentOut = path.join(tempDir, 'current_output');
  const compareOut = path.join(tempDir, 'compare_output');

  const scriptName = path.join('plugins', args.plugin + '.js');

  // export with current plugin script
  exportFixtures(
    currentOut,
    path.join(__dirname, '..', scriptName),
    path.join(__dirname, '..')
  );

  // get compare script and fixture files as archive
  const compareArchive = path.join(tempDir, 'compare.zip');
  try {
    childProcess.execSync(
      `git archive ${args.ref} ${scriptName} ${fixtures.join(' ')} -o ${compareArchive}`,
      {
        cwd: path.join(__dirname, '..'),
        encoding: 'utf-8'
      }
    );
  }
  catch (e) {
    console.error(colors.red('[Error]') + ` Failed downloading compare plugin script: ${e.message}`);
    process.exit(1);
  }

  // unzip archive
  const unzip = new Zip(fs.readFileSync(compareArchive));
  const unzipDir = path.join(tempDir, 'compareFiles');
  fs.mkdirSync(unzipDir);
  fs.mkdirSync(path.join(unzipDir, 'fixtures'));
  for (let file in unzip.files) {
    file = unzip.files[file];
    const filePath = path.join(unzipDir, file.name);

    if (!file.dir) {
      fs.writeFileSync(filePath, file.asNodeBuffer());
    }
    else if (!fs.existsSync(filePath)) {
      fs.mkdirSync(filePath);
    }
  }
  del.sync(compareArchive, {force: true});

  // also copy current defaults.js (will be needed by plugin script)
  fs.writeFileSync(
    path.join(unzipDir, 'fixtures', 'defaults.js'),
    fs.readFileSync(path.join(fixtureDir, 'defaults.js'))
  );

  // export with compare plugin script
  exportFixtures(
    compareOut,
    path.join(unzipDir, scriptName),
    unzipDir
  );

  // make recursive diff
  diffDir('');
  findAddedFiles('');
  console.log('Done.');

  if (resolve) {
    resolve(outputData);
  }

  // diffs recursively compare output vs. current output
  function diffDir(subdir) {
    const dir = path.join(compareOut, subdir);
    const files = fs.readdirSync(dir);

    for (const file of files) {
      const filePath = path.join(subdir, file);
      const filePathCompare = path.join(compareOut, filePath);
      const filePathCurrent = path.join(currentOut, filePath);<commit_msg>Handle default.js changes in plugin diff (#123)<commit_after>const fs = require('fs');
const path = require('path');
const childProcess = require('child_process');
const del = require('node-delete');
const Zip = require('node-zip');
const colors = require('colors');
const diff = require('diff');

module.exports = function(args, resolve) {
  let outputData = {
    removedFiles: [],
    addedFiles: [],
    changedFiles: {}
  };

  // get manufacturer and fixture data later used by export plugins
  const fixtureDir = path.join(__dirname, '..', 'fixtures');
  const manufacturers = JSON.parse(fs.readFileSync(
    path.join(fixtureDir, 'manufacturers.json')
  ));
  let library = [];
  let fixtures = [];
  for (let fixture of args.fixtures) {
    // fixture path is relative to user's working directory
    const fixPath = path.join(process.env.PWD, fixture);
    library.push({
      manufacturerKey: path.basename(path.dirname(fixPath)),
      fixtureKey: path.basename(fixPath, path.extname(fixPath))
    });
    fixtures.push(path.join('fixtures', path.relative(fixtureDir, fixPath)));
  }

  console.log();
  console.log('## Diffing plugin output');
  console.log(`# plugin: ${args.plugin}`);
  console.log(`# ref: ${args.ref}`);
  console.log(`# fixtures: ${fixtures}`);
  console.log();

  // delete old temp folder and recreate it
  const tempDir = path.join(__dirname, '..', 'tmp');
  if (process.env.PWD.match(tempDir)) {
    console.error(colors.red('[Error]') + ' This script can\'t be run from inside the tmp directory.');
    process.exit(1);
  }
  del.sync([tempDir], {force: true});
  fs.mkdirSync(tempDir);
  const currentOut = path.join(tempDir, 'current_output');
  const compareOut = path.join(tempDir, 'compare_output');

  const scriptName = path.join('plugins', args.plugin + '.js');

  // export with current plugin script
  exportFixtures(
    currentOut,
    path.join(__dirname, '..', scriptName),
    path.join(__dirname, '..')
  );

  // get compare script and fixture files as archive
  const compareArchive = path.join(tempDir, 'compare.zip');
  try {
    childProcess.execSync(
      `git archive ${args.ref} ${scriptName} ${fixtures.join(' ')} fixtures/defaults.js -o ${compareArchive}`,
      {
        cwd: path.join(__dirname, '..'),
        encoding: 'utf-8'
      }
    );
  }
  catch (e) {
    console.error(colors.red('[Error]') + ` Failed downloading compare plugin script: ${e.message}`);
    process.exit(1);
  }

  // unzip archive
  const unzip = new Zip(fs.readFileSync(compareArchive));
  const unzipDir = path.join(tempDir, 'compareFiles');
  fs.mkdirSync(unzipDir);
  for (let file in unzip.files) {
    file = unzip.files[file];
    const filePath = path.join(unzipDir, file.name);

    if (file.dir) {
      fs.mkdirSync(filePath);
    }
    else {
      fs.writeFileSync(filePath, file.asNodeBuffer());
    }
  }
  del.sync(compareArchive, {force: true});

  // export with compare plugin script
  exportFixtures(
    compareOut,
    path.join(unzipDir, scriptName),
    unzipDir
  );

  // make recursive diff
  diffDir('');
  findAddedFiles('');
  console.log('Done.');

  if (resolve) {
    resolve(outputData);
  }

  // diffs recursively compare output vs. current output
  function diffDir(subdir) {
    const dir = path.join(compareOut, subdir);
    const files = fs.readdirSync(dir);

    for (const file of files) {
      const filePath = path.join(subdir, file);
      const filePathCompare = path.join(compareOut, filePath);
      const filePathCurrent = path.join(currentOut, filePath);
{'fraction_non_alphanumeric': 0.1552617230098146, 'fraction_numerical': 0.0012268266085059978, 'mean_word_length': 3.591364205256571, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5364975', 'n_tokens_mistral': 2359, 'n_tokens_neox': 2309, 'n_words': 552}