<commit_before>// Note: You must restart bin/webpack-dev-server for changes to take effect

/* eslint global-require: 0 */

const merge = require('webpack-merge');
const CompressionPlugin = require('compression-webpack-plugin');
const { StatsWriterPlugin } = require('webpack-stats-plugin');
const sharedConfig = require('./shared.js');

module.exports = merge(sharedConfig, {
  mode: 'production',
  devtool: 'source-map',
  stats: 'normal',

  plugins: [
    new CompressionPlugin({
      test: /\.(js|css|html|json|ico|svg|eot|otf|ttf)$/,
    }),

    // Write out dependencies list for audit purposes
    new StatsWriterPlugin({
      filename: "webpack-modules-manifest.json",
      fields: ["modules"],

      transform(data) {
        // recursively flatten nested modules
        const transformModule  = m  => m.modules ? transformModules(m.modules) : m.name;
        const transformModules = ms => ms.flatMap(m => transformModule(m));

        var modules = transformModules(data.modules).sort();
        modules = [...new Set(modules)]; // uniq
        return JSON.stringify(modules, null, 2);
      }
    })
  ],
});<commit_msg>Cross reference the webpack manifest with the actual node_modules<commit_after>// Note: You must restart bin/webpack-dev-server for changes to take effect

/* eslint global-require: 0 */

const merge = require('webpack-merge');
const CompressionPlugin = require('compression-webpack-plugin');
const sharedConfig = require('./shared.js');

const { StatsWriterPlugin } = require('webpack-stats-plugin');
const fs = require('fs');
const sortBy = require('lodash/sortBy');

module.exports = merge(sharedConfig, {
  mode: 'production',
  devtool: 'source-map',
  stats: 'normal',

  plugins: [
    new CompressionPlugin({
      test: /\.(js|css|html|json|ico|svg|eot|otf|ttf)$/,
    }),

    // Write out dependencies list for audit purposes
    new StatsWriterPlugin({
      filename: "webpack-modules-manifest.json",
      fields: ["modules"],

      transform(data) {
        // recursively flatten nested modules
        const transformModule  = m  => m.modules ? transformModules(m.modules) : m.name.split("!").pop();
        const transformModules = ms => ms.flatMap(m => transformModule(m));

        var modules = transformModules(data.modules)
        modules = [...new Set(modules)]; // uniq

        // find the package.json file for each module
        modules = modules.filter(m => m.includes("node_modules"));
        var packagePaths = modules.map(m => {
          var match = m.match(/(.*node_modules\/)([^\/]+)(\/[^\/]+)?/);
          var path = [
            `${match[1]}${match[2]}/package.json`,
            `${match[1]}${match[2]}${match[3]}/package.json`,
          ].find(p => fs.existsSync(p));

          if (path == null) {
            console.warn(`[webpack-manifest] WARN: Unable to find a package.json for ${m}`);
          }

          return path;
        })
        packagePaths = packagePaths.filter(p => p != null);
        packagePaths = [...new Set(packagePaths)]; // uniq

        // extract relevant package data from the package.json
        var packages = packagePaths.map(p => {
          var content = fs.readFileSync(p);
          var package = JSON.parse(content);
          return {
            name: package.name,
            license: package.license,
            version: package.version,
            location: p
          }
        });
        packages = sortBy(packages, ['name', 'version']);

        return JSON.stringify(packages, null, 2);
      }
    })
  ],
});
{'fraction_non_alphanumeric': 0.15291467192340186, 'fraction_numerical': 0.0025344973246972683, 'mean_word_length': 2.688473520249221, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5081554', 'n_tokens_mistral': 1022, 'n_tokens_neox': 997, 'n_words': 245}