<commit_before>var pathModule = require('path');
var Promise = require('bluebird');
var urlTools = require('urltools');
var estraverse = require('estraverse');
var readPkgUp = require('read-pkg-up');

module.exports = function (queryObj, options) {
    options = options || {};
    return function bundleWebpack(assetGraph) {
        var webpack;
        try {
            webpack = require('webpack');
        } catch (e) {
            if (options.configPath) {
                assetGraph.emit('error', new Error('Webpack config path given (' + options.configPath + '), but webpack itself could not be found. Please install it in your project: ' + e.stack));
                return;
            }
        }

        var triedConfigPaths = [];

        function loadWebpackConfig() {
            return Promise.resolve().then(function () {
                // First try to look for the config in the assetGraph root:
                var configPath = options.configPath || pathModule.resolve(urlTools.fileUrlToFsPath(assetGraph.root), 'webpack.config.js');
                triedConfigPaths.push(configPath);
                try {
                    return require.main.require(configPath);
                } catch (err) {
                    if (err.code !== 'MODULE_NOT_FOUND' || options.configPath) {
                        throw err;
                    }
                    // webpack.config.js was not found in the assetgraph's root.
                    // Look for it in the directory that contains the nearest package.json:
                    return readPkgUp()
                    .then(function (readPkgUpResult) {
                        if (readPkgUpResult.path) {
                            var alternativeConfigPath = pathModule.resolve(pathModule.dirname(readPkgUpResult.path), 'webpack.config.js');
                            if (alternativeConfigPath !== configPath) {
                                configPath = alternativeConfigPath;
                                triedConfigPaths.push(configPath);
                                return require.main.require(configPath);
                            }
                        }
                        throw new Error('Could not load webpack config');
                    });
                }
            });
        }
        return loadWebpackConfig().then(function (config) {
            config.context = config.context || urlTools.fileUrlToFsPath(assetGraph.root);
            config.output = config.output || {};
            if (typeof config.output.path === 'string') {
                config.output.path = pathModule.resolve(config.context, config.output.path);
            }
            config.devtool = config.devtool || 'source-map';
            config.output.devtoolModuleFilenameTemplate = config.output.devtoolModuleFilenameTemplate || 'file://[absolute-resource-path]';
            var basePath = config.output.path;
            if (config.output.publicPath) {
                basePath = pathModule.resolve(config.context, config.output.publicPath.replace(/^\//, ''));
            }

            var plugins = Array.isArray(config.plugins) ? config.plugins : [];

            plugins.forEach(function (plugin) {
                if (plugin.constructor.name === 'UglifyJsPlugin' && (plugin.options.mangle || plugin.options.compress)) {
                    plugin.options.mangle = plugin.options.compress = false;
                    assetGraph.emit('info', new Error('UglifyJsPlugin detected, turning off mangling and compression so outgoing relations can be detected. This is fine for assetgraph-builder, which will run UglifyJs later anyway'));
                }
            });

            var hasHtmlWebpackPlugin = plugins.some(function (plugin) {
                return plugin.constructor.name === 'HtmlWebpackPlugin';
            });

            if (!hasHtmlWebpackPlugin) {
                // HtmlWebpackPlugin is not in the picture, so we can assume that the
                // relevant entry points are referenced by assets that we have
                // in the graph. Only build the actually referenced entry points:
                var seenReferencedEntryPoints = false;
                var potentialBundlePaths = assetGraph.findRelations({
                    to: {
                        url: /\.js(?:\?|$)/
                    }
                }, true).map(function (relation) {
                    return urlTools.fileUrlToFsPath(assetGraph.resolveUrl(relation.from.nonInlineAncestor.url, relation.to.url));
                });

                var createAbsolutePath = function (name) {
                    return pathModule.resolve(
                        basePath,
                        name ? config.output.filename.replace(/\[name]/g, name) : config.output.filename
                    );
                };

                if (config.entry && typeof config.entry === 'object' && !Array.isArray(config.entry)) {
                    config.entry = Object.keys(config.entry).reduce(function (entryPointsToBeBuilt, entryPointName) {
                        if (potentialBundlePaths.indexOf(createAbsolutePath(entryPointName)) !== -1) {
                            entryPointsToBeBuilt[entryPointName] = config.entry[entryPointName];
                        }
                        return entryPointsToBeBuilt;
                    }, {});

                    seenReferencedEntryPoints = Object.keys(config.entry).length > 0;
                } else {
                    // array or string
                    if (potentialBundlePaths.indexOf(createAbsolutePath()) !== -1) {
                        seenReferencedEntryPoints = true;
                    }
                }

                if (!seenReferencedEntryPoints) {
                    assetGraph.emit('info', new Error('No matching webpack bundles found, skipping'));
                    return;
                }
            }

            var compiler = webpack(config);
            var outputFileSystem = new webpack.MemoryOutputFileSystem();
            compiler.outputFileSystem = outputFileSystem;
            return Promise.fromNode(function (cb) {
                compiler.run(cb);
            }).then(function (stats) {
                var statsObj = stats.toJson({assets: true});
                var existingAssetsWereReplaced;
                var assetInfos = [];
                var urlByAssetName = {};
                statsObj.assets.forEach(function (asset) {
                    var url = urlTools.fsFilePathToFileUrl(pathModule.resolve(basePath, asset.name));
                    var outputPath = urlTools.fsFilePathToFileUrl(pathModule.resolve(config.output.path, asset.name));
                    assetInfos.push({
                        url: url,
                        output: outputPath,
                        name: asset.name
                    });
                    urlByAssetName[asset.name] = url;
                    var existingAsset = assetGraph.findAssets({url: url})[0];
                    if (existingAsset) {
                        assetGraph.emit('info', new Error('Replacing previously built artifact: ' + url));
                        existingAssetsWereReplaced = true;
                        existingAsset.incomingRelations.forEach(function (relation) {
                            relation.to = {url: url};
                        });
                        assetGraph.removeAsset(existingAsset);
                    }
                });
                assetGraph.emit('info', stats.toString({colors: true}));
                if (existingAssetsWereReplaced) {
                    assetGraph.emit('info', new Error('Please remove ' + config.output.path + ' before building with assetgraph to speed up the process'));
                }
                return assetGraph.loadAssets(assetInfos.map(function (url) {
                    return {
                        isInitial: /\.html$/.test(url.url),
                        url: url.url,
                        rawSrc: outputFileSystem.readFileSync(urlTools.fileUrlToFsPath(url.output))
                    };
                })).then(function () {
                    // Pick up CSS assets and attach them to each entry point
                    Object.keys(statsObj.assetsByChunkName).forEach(function (chunkName) {
                        var assets = statsObj.assetsByChunkName[chunkName];
                        var javaScriptAssets = assets.filter(function (asset) { return /\.js$/.test(asset); });
                        var cssAssets = assets.filter(function (asset) { return /\.css$/.test(asset); });
                        if (javaScriptAssets.length === 1 && cssAssets.length > 0) {
                            var entryPointRelations = assetGraph.findRelations(function (relation) {
                                if (relation.type !== 'HtmlScript' || !relation.to || !relation.to.url) {
                                    return;
                                }
                                var absoluteUrl = assetGraph.resolveUrl(relation.from.nonInlineAncestor.url, relation.to.url);
                                return absoluteUrl === urlByAssetName[javaScriptAssets[0]];
                            }, true);
                            entryPointRelations.forEach(function (entryPointRelation) {
                                cssAssets.forEach(function (cssAsset) {
                                    var htmlStyle = new assetGraph.HtmlStyle({
                                        hrefType: 'rootRelative',
                                        to: assetGraph.findAssets({url: urlByAssetName[cssAsset]})[0]
                                    });
                                    htmlStyle.attach(entryPointRelation.from, 'last');
                                });
                            });
                        }
                    });

                    // Fix up sources: [...] urls in source maps:
                    assetGraph.findRelations({
                        type: 'SourceMapSource',
                        from: { url: assetInfos.map(function (url) { return url.url; }) }
                    }, true).forEach(function (relation) {
                        if (relation.href.indexOf('file:///') === 0 && relation.href.indexOf(assetGraph.root) !== 0) {
                            // Probably a file from node_modules outside of assetGraph.root.
                            // Make it relative so that it points out of the webroot instead of
                            // ending up as an absolute file:// url:
                            relation.hrefType = 'relative';
                        } else if (relation.href.indexOf('file://webpack/') === 0) {
                            // Some fake webpack url, eg. "webpack/bootstrap 5c89cbd3893270d2912a"
                            // that got file:// prepended due to devtoolFallbackModuleFilenameTemplate
                            // Strip the file:// prefix and make it relative so that it sort of
                            // looks like the webpack output:
                            relation.to.url = relation.to.url.substr('file://'.length);
                            relation.hrefType = 'relative';
                            relation.refreshHref();
                        } else {
                            relation.hrefType = 'rootRelative';
                        }
                    });

                    assetGraph.findAssets({
                        type: 'JavaScript',
                        url: assetInfos.map(function (url) { return url.url; })
                    }).forEach(function (asset) {
                        var replacementsMade = false;
                        var requireEnsureAstTemplate;
                        estraverse.traverse(asset.parseTree, {
                            enter: function (node) {
                                // Detect the __webpack_require__.e = function requireEnsure(chunkId, callback) { ... declaration at the top of a bundle
                                if (node.type === 'AssignmentExpression' &&
                                    node.operator === '=' &&
                                    node.left.type === 'MemberExpression' &&
                                    node.left.object.type === 'Identifier' &&
                                    node.left.object.name === '__webpack_require__' &&
                                    node.left.property.type === 'Identifier' &&
                                    node.left.property.name === 'e' &&
                                    node.right.type === 'FunctionExpression' &&
                                    node.right.id &&
                                    node.right.id.type === 'Identifier' &&<commit_msg>bundleWebpack: Simplify source map postprocessing now that urltools.buildRootRelativeUrl handles file: urls pointing out of the root<commit_after>var pathModule = require('path');
var Promise = require('bluebird');
var urlTools = require('urltools');
var estraverse = require('estraverse');
var readPkgUp = require('read-pkg-up');

module.exports = function (queryObj, options) {
    options = options || {};
    return function bundleWebpack(assetGraph) {
        var webpack;
        try {
            webpack = require('webpack');
        } catch (e) {
            if (options.configPath) {
                assetGraph.emit('error', new Error('Webpack config path given (' + options.configPath + '), but webpack itself could not be found. Please install it in your project: ' + e.stack));
                return;
            }
        }

        var triedConfigPaths = [];

        function loadWebpackConfig() {
            return Promise.resolve().then(function () {
                // First try to look for the config in the assetGraph root:
                var configPath = options.configPath || pathModule.resolve(urlTools.fileUrlToFsPath(assetGraph.root), 'webpack.config.js');
                triedConfigPaths.push(configPath);
                try {
                    return require.main.require(configPath);
                } catch (err) {
                    if (err.code !== 'MODULE_NOT_FOUND' || options.configPath) {
                        throw err;
                    }
                    // webpack.config.js was not found in the assetgraph's root.
                    // Look for it in the directory that contains the nearest package.json:
                    return readPkgUp()
                    .then(function (readPkgUpResult) {
                        if (readPkgUpResult.path) {
                            var alternativeConfigPath = pathModule.resolve(pathModule.dirname(readPkgUpResult.path), 'webpack.config.js');
                            if (alternativeConfigPath !== configPath) {
                                configPath = alternativeConfigPath;
                                triedConfigPaths.push(configPath);
                                return require.main.require(configPath);
                            }
                        }
                        throw new Error('Could not load webpack config');
                    });
                }
            });
        }
        return loadWebpackConfig().then(function (config) {
            config.context = config.context || urlTools.fileUrlToFsPath(assetGraph.root);
            config.output = config.output || {};
            if (typeof config.output.path === 'string') {
                config.output.path = pathModule.resolve(config.context, config.output.path);
            }
            config.devtool = config.devtool || 'source-map';
            config.output.devtoolModuleFilenameTemplate = config.output.devtoolModuleFilenameTemplate || 'file://[absolute-resource-path]';
            var basePath = config.output.path;
            if (config.output.publicPath) {
                basePath = pathModule.resolve(config.context, config.output.publicPath.replace(/^\//, ''));
            }

            var plugins = Array.isArray(config.plugins) ? config.plugins : [];

            plugins.forEach(function (plugin) {
                if (plugin.constructor.name === 'UglifyJsPlugin' && (plugin.options.mangle || plugin.options.compress)) {
                    plugin.options.mangle = plugin.options.compress = false;
                    assetGraph.emit('info', new Error('UglifyJsPlugin detected, turning off mangling and compression so outgoing relations can be detected. This is fine for assetgraph-builder, which will run UglifyJs later anyway'));
                }
            });

            var hasHtmlWebpackPlugin = plugins.some(function (plugin) {
                return plugin.constructor.name === 'HtmlWebpackPlugin';
            });

            if (!hasHtmlWebpackPlugin) {
                // HtmlWebpackPlugin is not in the picture, so we can assume that the
                // relevant entry points are referenced by assets that we have
                // in the graph. Only build the actually referenced entry points:
                var seenReferencedEntryPoints = false;
                var potentialBundlePaths = assetGraph.findRelations({
                    to: {
                        url: /\.js(?:\?|$)/
                    }
                }, true).map(function (relation) {
                    return urlTools.fileUrlToFsPath(assetGraph.resolveUrl(relation.from.nonInlineAncestor.url, relation.to.url));
                });

                var createAbsolutePath = function (name) {
                    return pathModule.resolve(
                        basePath,
                        name ? config.output.filename.replace(/\[name]/g, name) : config.output.filename
                    );
                };

                if (config.entry && typeof config.entry === 'object' && !Array.isArray(config.entry)) {
                    config.entry = Object.keys(config.entry).reduce(function (entryPointsToBeBuilt, entryPointName) {
                        if (potentialBundlePaths.indexOf(createAbsolutePath(entryPointName)) !== -1) {
                            entryPointsToBeBuilt[entryPointName] = config.entry[entryPointName];
                        }
                        return entryPointsToBeBuilt;
                    }, {});

                    seenReferencedEntryPoints = Object.keys(config.entry).length > 0;
                } else {
                    // array or string
                    if (potentialBundlePaths.indexOf(createAbsolutePath()) !== -1) {
                        seenReferencedEntryPoints = true;
                    }
                }

                if (!seenReferencedEntryPoints) {
                    assetGraph.emit('info', new Error('No matching webpack bundles found, skipping'));
                    return;
                }
            }

            var compiler = webpack(config);
            var outputFileSystem = new webpack.MemoryOutputFileSystem();
            compiler.outputFileSystem = outputFileSystem;
            return Promise.fromNode(function (cb) {
                compiler.run(cb);
            }).then(function (stats) {
                var statsObj = stats.toJson({assets: true});
                var existingAssetsWereReplaced;
                var assetInfos = [];
                var urlByAssetName = {};
                statsObj.assets.forEach(function (asset) {
                    var url = urlTools.fsFilePathToFileUrl(pathModule.resolve(basePath, asset.name));
                    var outputPath = urlTools.fsFilePathToFileUrl(pathModule.resolve(config.output.path, asset.name));
                    assetInfos.push({
                        url: url,
                        output: outputPath,
                        name: asset.name
                    });
                    urlByAssetName[asset.name] = url;
                    var existingAsset = assetGraph.findAssets({url: url})[0];
                    if (existingAsset) {
                        assetGraph.emit('info', new Error('Replacing previously built artifact: ' + url));
                        existingAssetsWereReplaced = true;
                        existingAsset.incomingRelations.forEach(function (relation) {
                            relation.to = {url: url};
                        });
                        assetGraph.removeAsset(existingAsset);
                    }
                });
                assetGraph.emit('info', stats.toString({colors: true}));
                if (existingAssetsWereReplaced) {
                    assetGraph.emit('info', new Error('Please remove ' + config.output.path + ' before building with assetgraph to speed up the process'));
                }
                return assetGraph.loadAssets(assetInfos.map(function (url) {
                    return {
                        isInitial: /\.html$/.test(url.url),
                        url: url.url,
                        rawSrc: outputFileSystem.readFileSync(urlTools.fileUrlToFsPath(url.output))
                    };
                })).then(function () {
                    // Pick up CSS assets and attach them to each entry point
                    Object.keys(statsObj.assetsByChunkName).forEach(function (chunkName) {
                        var assets = statsObj.assetsByChunkName[chunkName];
                        var javaScriptAssets = assets.filter(function (asset) { return /\.js$/.test(asset); });
                        var cssAssets = assets.filter(function (asset) { return /\.css$/.test(asset); });
                        if (javaScriptAssets.length === 1 && cssAssets.length > 0) {
                            var entryPointRelations = assetGraph.findRelations(function (relation) {
                                if (relation.type !== 'HtmlScript' || !relation.to || !relation.to.url) {
                                    return;
                                }
                                var absoluteUrl = assetGraph.resolveUrl(relation.from.nonInlineAncestor.url, relation.to.url);
                                return absoluteUrl === urlByAssetName[javaScriptAssets[0]];
                            }, true);
                            entryPointRelations.forEach(function (entryPointRelation) {
                                cssAssets.forEach(function (cssAsset) {
                                    var htmlStyle = new assetGraph.HtmlStyle({
                                        hrefType: 'rootRelative',
                                        to: assetGraph.findAssets({url: urlByAssetName[cssAsset]})[0]
                                    });
                                    htmlStyle.attach(entryPointRelation.from, 'last');
                                });
                            });
                        }
                    });

                    // Fix up sources: [...] urls in source maps:
                    assetGraph.findRelations({
                        type: 'SourceMapSource',
                        from: { url: assetInfos.map(function (url) { return url.url; }) }
                    }, true).forEach(function (relation) {
                        if (relation.href.indexOf('file://webpack/') === 0) {
                            // Some fake webpack url, eg. "webpack/bootstrap 5c89cbd3893270d2912a"
                            // that got file:// prepended due to devtoolFallbackModuleFilenameTemplate
                            // Strip the file:// prefix and make it relative so that it sort of
                            // looks like the webpack output:
                            relation.to.url = relation.to.url.substr('file://'.length);
                            relation.hrefType = 'relative';
                            relation.refreshHref();
                        } else {
                            relation.hrefType = 'rootRelative';
                        }
                    });

                    assetGraph.findAssets({
                        type: 'JavaScript',
                        url: assetInfos.map(function (url) { return url.url; })
                    }).forEach(function (asset) {
                        var replacementsMade = false;
                        var requireEnsureAstTemplate;
                        estraverse.traverse(asset.parseTree, {
                            enter: function (node) {
                                // Detect the __webpack_require__.e = function requireEnsure(chunkId, callback) { ... declaration at the top of a bundle
                                if (node.type === 'AssignmentExpression' &&
                                    node.operator === '=' &&
                                    node.left.type === 'MemberExpression' &&
                                    node.left.object.type === 'Identifier' &&
                                    node.left.object.name === '__webpack_require__' &&
                                    node.left.property.type === 'Identifier' &&
                                    node.left.property.name === 'e' &&
                                    node.right.type === 'FunctionExpression' &&
                                    node.right.id &&
                                    node.right.id.type === 'Identifier' &&
{'fraction_non_alphanumeric': 0.09843160611475084, 'fraction_numerical': 0.0019058963668850506, 'mean_word_length': 1.3257918552036199, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 42, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1186820', 'n_tokens_mistral': 5877, 'n_tokens_neox': 5464, 'n_words': 1213}