<commit_before>#!/usr/bin/node

const path = require(`path`);

const diffPluginOutputs = require(`../../lib/diff-plugin-outputs.js`);
const exportPlugins = require(`../../plugins/plugins.json`).exportPlugins.filter(pluginKey => pluginKey !== `ofl`); // don't diff (essentially) the source files
const pullRequest = require(`./pull-request.js`);

require(`../../lib/load-env-file.js`);

const testFixtures = require(`../test-fixtures.json`).map(
  fixture => `${fixture.man}/${fixture.key}`
);

/**
 * @typedef {object} Task
 * @property {string} manFix
 * @property {string} pluginKey
 */

// generate diff tasks describing the diffed plugins, fixtures and the reason for diffing (which component has changed)
pullRequest.checkEnv()
  .catch(error => {
    console.error(error);
    process.exit(0); // if the environment is not correct, just exit without failing
  })
  .then(() => pullRequest.init())
  .then(prData => pullRequest.fetchChangedComponents())
  .then(changedComponents => {
    const usablePlugins = exportPlugins.filter(plugin => !changedComponents.added.exports.includes(plugin));
    const addedFixtures = changedComponents.added.fixtures.map(([man, key]) => `${man}/${key}`);
    const usableTestFixtures = testFixtures.filter(testFixture => !addedFixtures.includes(testFixture));

    /** @type {Array.<Task>} */
    const tasks = getTasksForModel()
      .concat(getTasksForPlugins())
      .concat(getTasksForFixtures())
      .filter((task, index, arr) => {
        const firstEqualTask = arr.find(otherTask =>
          task.manFix === otherTask.manFix &&
          task.pluginKey === otherTask.pluginKey
        );

        // remove duplicates
        return task === firstEqualTask;
      })
      .sort((a, b) => {
        const manFixCompare = a.manFix.localeCompare(b.manFix);
        const pluginCompare = a.pluginKey.localeCompare(b.pluginKey);

        if (manFixCompare !== 0) {
          return manFixCompare;
        }

        return pluginCompare;
      });

    return tasks;

    /**
     * @returns {!Array.<Task>} What export diff tasks have to be done due to changes in the model. May be empty.
     */
    function getTasksForModel() {
      let tasks = [];

      if (changedComponents.added.model ||
        changedComponents.modified.model ||
        changedComponents.renamed.model ||
        changedComponents.removed.model) {

        for (const manFix of usableTestFixtures) {
          tasks = tasks.concat(usablePlugins.map(pluginKey => ({
            manFix,
            pluginKey
          })));
        }
      }

      return tasks;
    }

    /**
     * @returns {!Array.<Task>} What export diff tasks have to be done due to changes in plugins. May be empty.
     */
    function getTasksForPlugins() {
      let tasks = [];

      const changedPlugins = changedComponents.added.exports.concat(changedComponents.modified.exports);

      for (const changedPlugin of changedPlugins) {
        tasks = tasks.concat(usableTestFixtures.map(manFix => ({
          manFix,
          pluginKey: changedPlugin
        })));
      }

      return tasks;
    }

    /**
     * @returns {!Array.<Task>} What export diff tasks have to be done due to changes in fixtures. May be empty.
     */
    function getTasksForFixtures() {
      let tasks = [];

      const fixtures = changedComponents.added.fixtures.concat(changedComponents.modified.fixtures, changedComponents.renamed.fixtures);

      for (const [manKey, fixKey] of fixtures) {
        tasks = tasks.concat(usablePlugins.map(pluginKey => ({
          manFix: `${manKey}/${fixKey}`,
          pluginKey
        })));
      }

      return tasks;
    }
  })
  .then(tasks => {
    if (tasks.length === 0) {
      return pullRequest.updateComment({
        filename: path.relative(path.join(__dirname, `../../`), __filename),
        name: `Plugin export diff`,
        lines: []
      });
    }

    let lines = [
      `You can run view your uncommited changes in plugin exports manually by executing:`,
      `\`$ node cli/diff-plugin-outputs.js -p <plugin name> <fixtures>\``,
      ``
    ];

    const tooLongMessage = `:warning: The output of the script is too long to fit in this comment, please run it yourself locally or download the raw Travis log.`;

    for (const task of tasks) {
      const taskResultLines = performTask(task);

      // GitHub's offical maximum comment length is 2^16=65536, but it's actually 2^18=262144.
      // We keep 2144 characters extra space as we don't count the comment header (added by our pull request module).
      if (lines.concat(taskResultLines).concat(tooLongMessage).join(`\r\n`).length > 260000) {
        lines.push(tooLongMessage);
        break;
      }

      lines = lines.concat(taskResultLines);
    }

    return pullRequest.updateComment({<commit_msg>Export diff fixes (#502)<commit_after>#!/usr/bin/node

const path = require(`path`);

const diffPluginOutputs = require(`../../lib/diff-plugin-outputs.js`);
const exportPlugins = require(`../../plugins/plugins.json`).exportPlugins.filter(pluginKey => pluginKey !== `ofl`); // don't diff (essentially) the source files
const pullRequest = require(`./pull-request.js`);

require(`../../lib/load-env-file.js`);

const testFixtures = require(`../test-fixtures.json`).map(
  fixture => `${fixture.man}/${fixture.key}`
);

/**
 * @typedef {object} Task
 * @property {string} manFix
 * @property {string} pluginKey
 */

// generate diff tasks describing the diffed plugins, fixtures and the reason for diffing (which component has changed)
pullRequest.checkEnv()
  .catch(error => {
    console.error(error);
    process.exit(0); // if the environment is not correct, just exit without failing
  })
  .then(() => pullRequest.init())
  .then(prData => pullRequest.fetchChangedComponents())
  .then(changedComponents => {
    const usablePlugins = exportPlugins.filter(plugin => !changedComponents.added.exports.includes(plugin));
    const addedFixtures = changedComponents.added.fixtures.map(([man, key]) => `${man}/${key}`);
    const usableTestFixtures = testFixtures.filter(testFixture => !addedFixtures.includes(testFixture));

    /** @type {Array.<Task>} */
    const tasks = getTasksForModel()
      .concat(getTasksForPlugins())
      .concat(getTasksForFixtures())
      .filter((task, index, arr) => {
        const firstEqualTask = arr.find(otherTask =>
          task.manFix === otherTask.manFix &&
          task.pluginKey === otherTask.pluginKey
        );

        // remove duplicates
        return task === firstEqualTask;
      })
      .sort((a, b) => {
        const manFixCompare = a.manFix.localeCompare(b.manFix);
        const pluginCompare = a.pluginKey.localeCompare(b.pluginKey);

        if (manFixCompare !== 0) {
          return manFixCompare;
        }

        return pluginCompare;
      });

    return tasks;

    /**
     * @returns {!Array.<Task>} What export diff tasks have to be done due to changes in the model. May be empty.
     */
    function getTasksForModel() {
      let tasks = [];

      if (changedComponents.added.model ||
        changedComponents.modified.model ||
        changedComponents.renamed.model ||
        changedComponents.removed.model) {

        for (const manFix of usableTestFixtures) {
          tasks = tasks.concat(usablePlugins.map(pluginKey => ({
            manFix,
            pluginKey
          })));
        }
      }

      return tasks;
    }

    /**
     * @returns {!Array.<Task>} What export diff tasks have to be done due to changes in plugins. May be empty.
     */
    function getTasksForPlugins() {
      let tasks = [];

      const changedPlugins = changedComponents.added.exports.concat(changedComponents.modified.exports);

      for (const changedPlugin of changedPlugins) {
        tasks = tasks.concat(usableTestFixtures.map(manFix => ({
          manFix,
          pluginKey: changedPlugin
        })));
      }

      return tasks;
    }

    /**
     * @returns {!Array.<Task>} What export diff tasks have to be done due to changes in fixtures. May be empty.
     */
    function getTasksForFixtures() {
      let tasks = [];

      for (const [manKey, fixKey] of changedComponents.modified.fixtures) {
        tasks = tasks.concat(usablePlugins.map(pluginKey => ({
          manFix: `${manKey}/${fixKey}`,
          pluginKey
        })));
      }

      return tasks;
    }
  })
  .then(tasks => {
    if (tasks.length === 0) {
      return pullRequest.updateComment({
        filename: path.relative(path.join(__dirname, `../../`), __filename),
        name: `Plugin export diff`,
        lines: []
      });
    }

    let lines = [
      `You can run view your uncommited changes in plugin exports manually by executing:`,
      `\`$ node cli/diff-plugin-outputs.js -p <plugin name> <fixtures>\``,
      ``
    ];

    const tooLongMessage = `:warning: The output of the script is too long to fit in this comment, please run it yourself locally!`;

    for (const task of tasks) {
      const taskResultLines = performTask(task);

      // GitHub's offical maximum comment length is 2^16=65536, but it's actually 2^18=262144.
      // We keep 2144 characters extra space as we don't count the comment header (added by our pull request module).
      if (lines.concat(taskResultLines).concat(tooLongMessage).join(`\r\n`).length > 260000) {
        lines.push(tooLongMessage);
        break;
      }

      lines = lines.concat(taskResultLines);
    }

    return pullRequest.updateComment({
{'fraction_non_alphanumeric': 0.13363473872358322, 'fraction_numerical': 0.006623909157817264, 'mean_word_length': 2.683965917893106, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1333436', 'n_tokens_mistral': 2780, 'n_tokens_neox': 2668, 'n_words': 688}