<commit_before>const webpack = require('webpack')
const CleanWebpackPlugin = require('clean-webpack-plugin')
const HtmlWebpackPlugin = require('html-webpack-plugin')
const OptimizeCssAssetsWebpackPlugin = require('optimize-css-assets-webpack-plugin')

const { StatsWriterPlugin } = require('webpack-stats-plugin');
const fs = require('fs');
const sortBy = require('lodash/sortBy');

const config = require('./webpack.dev.js')

const uniqueWebpackModules = (data) => {
  // recursively flatten nested modules
  const transformModule  = m  => m.modules ? transformModules(m.modules) : m.name.split("!").pop();
  const transformModules = ms => ms.flatMap(m => transformModule(m));

  var modules = transformModules(data.modules);
  modules = [...new Set(modules)]; // uniq

  return modules;
}

const packagesFromModules = (modules) => {
  // find the package.json file for each module
  modules = modules.filter(m => m.includes("node_modules"));
  var packagePaths = modules.map(m => {
    var match = m.match(/(.*node_modules\/)([^/]+)(\/[^/]+)?/);
    var path = [
      `${match[1]}${match[2]}/package.json`,
      `${match[1]}${match[2]}${match[3]}/package.json`,
    ].find(p => fs.existsSync(p));

    if (path == null) {
      console.warn(`[webpack-manifest] WARN: Unable to find a package.json for ${m}`);
    }

    return path;
  })
  packagePaths = packagePaths.filter(p => p != null);
  packagePaths = [...new Set(packagePaths)]; // uniq

  // extract relevant package data from the package.json
  var packages = packagePaths.map(p => {
    var content = fs.readFileSync(p);
    var pkg = JSON.parse(content);
    return {
      name: pkg.name,
      license: pkg.license,
      version: pkg.version,
      location: p
    }
  });
  packages = sortBy(packages, ['name', 'version']);

  return packages;
}

// Source maps suitable for production use
config.devtool = 'cheap-module-source-map'

config.plugins.push(
  new webpack.NoEmitOnErrorsPlugin(),

  new webpack.optimize.UglifyJsPlugin(),

  // Cleans previous build
  new CleanWebpackPlugin([config.output.path], {
    allowExternal: true
  }),

  // Replace index.html with correct base href for production use
  new HtmlWebpackPlugin({
    base: '/ui/service/',
    template: '../client/index.ejs',
    chunks: ['app']
  }),

  new OptimizeCssAssetsWebpackPlugin(),

  // Write out dependent modules list for audit purposes
  new StatsWriterPlugin({
    filename: "webpack_modules_manifest.json",
    fields: ["modules"],

    transform(data) {
      var modules = uniqueWebpackModules(data);
      return JSON.stringify(modules, null, 2);
    }
  }),

  // Write out dependent packages list for audit purposes
  new StatsWriterPlugin({
    filename: "webpack_packages_manifest.json",
    fields: ["modules"],

    transform(data) {
      var modules = uniqueWebpackModules(data);
      var packages = packagesFromModules(modules);
      return JSON.stringify(packages, null, 2);
    }
  })
)

module.exports = config
<commit_msg>Sort the webpack modules for consistent output<commit_after>const webpack = require('webpack')
const CleanWebpackPlugin = require('clean-webpack-plugin')
const HtmlWebpackPlugin = require('html-webpack-plugin')
const OptimizeCssAssetsWebpackPlugin = require('optimize-css-assets-webpack-plugin')

const { StatsWriterPlugin } = require('webpack-stats-plugin');
const fs = require('fs');
const sortBy = require('lodash/sortBy');

const config = require('./webpack.dev.js')

const uniqueWebpackModules = (data) => {
  // recursively flatten nested modules
  const transformModule  = m  => m.modules ? transformModules(m.modules) : m.name.split("!").pop();
  const transformModules = ms => ms.flatMap(m => transformModule(m));

  var modules = transformModules(data.modules).sort();
  modules = [...new Set(modules)]; // uniq

  return modules;
}

const packagesFromModules = (modules) => {
  // find the package.json file for each module
  modules = modules.filter(m => m.includes("node_modules"));
  var packagePaths = modules.map(m => {
    var match = m.match(/(.*node_modules\/)([^/]+)(\/[^/]+)?/);
    var path = [
      `${match[1]}${match[2]}/package.json`,
      `${match[1]}${match[2]}${match[3]}/package.json`,
    ].find(p => fs.existsSync(p));

    if (path == null) {
      console.warn(`[webpack-manifest] WARN: Unable to find a package.json for ${m}`);
    }

    return path;
  })
  packagePaths = packagePaths.filter(p => p != null);
  packagePaths = [...new Set(packagePaths)]; // uniq

  // extract relevant package data from the package.json
  var packages = packagePaths.map(p => {
    var content = fs.readFileSync(p);
    var pkg = JSON.parse(content);
    return {
      name: pkg.name,
      license: pkg.license,
      version: pkg.version,
      location: p
    }
  });
  packages = sortBy(packages, ['name', 'version']);

  return packages;
}

// Source maps suitable for production use
config.devtool = 'cheap-module-source-map'

config.plugins.push(
  new webpack.NoEmitOnErrorsPlugin(),

  new webpack.optimize.UglifyJsPlugin(),

  // Cleans previous build
  new CleanWebpackPlugin([config.output.path], {
    allowExternal: true
  }),

  // Replace index.html with correct base href for production use
  new HtmlWebpackPlugin({
    base: '/ui/service/',
    template: '../client/index.ejs',
    chunks: ['app']
  }),

  new OptimizeCssAssetsWebpackPlugin(),

  // Write out dependent modules list for audit purposes
  new StatsWriterPlugin({
    filename: "webpack_modules_manifest.json",
    fields: ["modules"],

    transform(data) {
      var modules = uniqueWebpackModules(data);
      return JSON.stringify(modules, null, 2);
    }
  }),

  // Write out dependent packages list for audit purposes
  new StatsWriterPlugin({
    filename: "webpack_packages_manifest.json",
    fields: ["modules"],

    transform(data) {
      var modules = uniqueWebpackModules(data);
      var packages = packagesFromModules(modules);
      return JSON.stringify(packages, null, 2);
    }
  })
)

module.exports = config

{'fraction_non_alphanumeric': 0.14988406757204373, 'fraction_numerical': 0.0023186485591255384, 'mean_word_length': 3.6453846153846152, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4082585', 'n_tokens_mistral': 1862, 'n_tokens_neox': 1817, 'n_words': 440}