<commit_before>const fs = require('fs');
const { promisify } = require('util');
const path = require('path');
const parse5 = require('parse5');
const nwmatcher = require('nwmatcher');
const css = require('css');
const prettyBytes = require('pretty-bytes');

const readFile = promisify(fs.readFile);

const treeAdapter = parse5.treeAdapters.htmlparser2;

const PLUGIN_NAME = 'critters-webpack-plugin';

const PARSE5_OPTS = {
  treeAdapter
};

/** Critters: Webpack Plugin Edition!
 *  @class
 *  @param {Object} options
 *  @param {Boolean} [options.external=true]  Fetch and inline critical styles from external stylesheets
 *  @param {Boolean} [options.async=false]    Convert critical-inlined external stylesheets to load asynchronously (via link rel="preload")
 *  @param {Boolean} [options.compress=true]  Compress resulting critical CSS
 */
module.exports = class CrittersWebpackPlugin {
  constructor(options) {
    this.options = options || {};
  }

  apply(compiler) {
    const outputPath = compiler.options.output.path;

    // hook into the compiler to get a Compilation instance...
    compiler.hooks.compilation.tap(PLUGIN_NAME, compilation => {
      // ... which is how we get an "after" hook into html-webpack-plugin's HTML generation.
      compilation.hooks.htmlWebpackPluginAfterHtmlProcessing.tapAsync(PLUGIN_NAME, (htmlPluginData, callback) => {
        // Parse the generated HTML in a DOM we can mutate
        const document = parse5.parse(htmlPluginData.html, PARSE5_OPTS);
        makeDomInteractive(document);

        // `external:false` skips processing of external sheets
        const externalSheets = this.options.external===false ? [] : document.querySelectorAll('link[rel="stylesheet"]');

        Promise.all(externalSheets.map(link => {
          const href = link.getAttribute('href');

          // skip network resources
          if (href.match(/^(https?:)?\/\//)) return Promise.resolve();

          // path on disk
          const filename = path.resolve(outputPath, href.replace(/^\//, ''));

          // try to find a matching asset by filename in webpack's output (not yet written to disk)
          const asset = compilation.assets[path.relative(outputPath, filename).replace(/^\.\//, '')];

          // wait for a disk read if we had to go to disk
          const promise = asset ? Promise.resolve(asset.source()) : readFile(filename);
          return promise.then(sheet => {
            // the reduced critical CSS gets injected into a new <style> tag
            const style = document.createElement('style');
            style.appendChild(document.createTextNode(sheet));
            link.parentNode.insertBefore(style, link.nextSibling);

            // drop a reference to the original URL onto the tag (used for reporting to console later)
            style.$$name = href;

            // the `async` option changes any critical'd <link rel="stylesheet"> tags to async-loaded equivalents
            if (this.options.async) {
              link.setAttribute('rel', 'preload');
              link.setAttribute('as', 'style');
              link.setAttribute('onload', "this.rel='stylesheet'");
            }
          });
        }))
          .then(() => {
            // go through all the style tags in the document and reduce them to only critical CSS
            const styles = document.$match.byTag('style');
            return Promise.all(styles.map(style => this.processStyle(style, document)));
          })
          .then(() => {
            const html = parse5.serialize(document, PARSE5_OPTS);
            callback(null, { html });
          })
          .catch((err) => {
            callback(err);
          });
      });
    });
  }

  processStyle(style, document) {
    let done = Promise.resolve();

    // basically `.textContent`
    let sheet = style.childNodes.length>0 && style.childNodes.map( node => node.nodeValue ).join('\n');

    // store a reference to the previous serialized stylesheet for reporting stats
    const before = sheet;

    // Skip empty stylesheets
    if (!sheet) return done;

    const ast = css.parse(sheet);

    // Walk all CSS rules, transforming unused rules to comments (which get removed)
    visit(ast, (rule) => {
      if (rule.type==='rule') {
        rule.selectors = rule.selectors.filter((sel) => {
          // Remove unknown pseudos as they break nwmatcher
          sel = sel.replace(/::?(?:[a-z-]+)([.[#~&^:*]|\s|\n|$)/gi, '$1');
          return document.querySelector(sel, document) != null;
        });
        // If there are no matched selectors, replace the rule with a comment.
        if (rule.selectors.length===0) {
          rule.type = 'comment';
          rule.comment = '';
          delete rule.selectors;
          delete rule.declarations;
        }
      }
<commit_msg>Hoist `embedLinkedStylesheet()` onto the class<commit_after>const fs = require('fs');
const { promisify } = require('util');
const path = require('path');
const parse5 = require('parse5');
const nwmatcher = require('nwmatcher');
const css = require('css');
const prettyBytes = require('pretty-bytes');

const readFile = promisify(fs.readFile);

const treeAdapter = parse5.treeAdapters.htmlparser2;

const PLUGIN_NAME = 'critters-webpack-plugin';

const PARSE5_OPTS = {
  treeAdapter
};

/** Critters: Webpack Plugin Edition!
 *  @class
 *  @param {Object} options
 *  @param {Boolean} [options.external=true]  Fetch and inline critical styles from external stylesheets
 *  @param {Boolean} [options.async=false]    Convert critical-inlined external stylesheets to load asynchronously (via link rel="preload")
 *  @param {Boolean} [options.compress=true]  Compress resulting critical CSS
 */
module.exports = class CrittersWebpackPlugin {
  constructor(options) {
    this.options = options || {};
  }

  apply(compiler) {
    const outputPath = compiler.options.output.path;

    // hook into the compiler to get a Compilation instance...
    compiler.hooks.compilation.tap(PLUGIN_NAME, compilation => {
      // ... which is how we get an "after" hook into html-webpack-plugin's HTML generation.
      compilation.hooks.htmlWebpackPluginAfterHtmlProcessing.tapAsync(PLUGIN_NAME, (htmlPluginData, callback) => {
        // Parse the generated HTML in a DOM we can mutate
        const document = parse5.parse(htmlPluginData.html, PARSE5_OPTS);
        makeDomInteractive(document);

        let externalStylesProcessed = Promise.resolve();

        // `external:false` skips processing of external sheets
        if (this.options.external!==false) {
          const externalSheets = document.querySelectorAll('link[rel="stylesheet"]');
          externalStylesProcessed = Promise.all(externalSheets.map(
            link => this.embedLinkedStylesheet(link, compilation, outputPath)
          ));
        }

        externalStylesProcessed
          .then(() => {
            // go through all the style tags in the document and reduce them to only critical CSS
            const styles = document.$match.byTag('style');
            return Promise.all(styles.map(style => this.processStyle(style, document)));
          })
          .then(() => {
            const html = parse5.serialize(document, PARSE5_OPTS);
            callback(null, { html });
          })
          .catch((err) => {
            callback(err);
          });
      });
    });
  }

  /** Inline the target stylesheet referred to by a <link rel="stylesheet"> (assuming it passes `options.filter`) */
  embedLinkedStylesheet(link, compilation, outputPath) {
    const href = link.getAttribute('href');
    const document = link.ownerDocument;

    // skip filtered resources, or network resources if no filter is provided
    if (this.urlFilter ? this.urlFilter(href) : href.match(/^(https?:)?\/\//)) return Promise.resolve();

    // path on disk
    const filename = path.resolve(outputPath, href.replace(/^\//, ''));

    // try to find a matching asset by filename in webpack's output (not yet written to disk)
    const asset = compilation.assets[path.relative(outputPath, filename).replace(/^\.\//, '')];

    // wait for a disk read if we had to go to disk
    const promise = asset ? Promise.resolve(asset.source()) : readFile(filename, 'utf8');
    return promise.then(sheet => {
    // the reduced critical CSS gets injected into a new <style> tag
      const style = document.createElement('style');
      style.appendChild(document.createTextNode(sheet));
      link.parentNode.insertBefore(style, link.nextSibling);

      // drop a reference to the original URL onto the tag (used for reporting to console later)
      style.$$name = href;

      // the `async` option changes any critical'd <link rel="stylesheet"> tags to async-loaded equivalents
      if (this.options.async) {
        link.setAttribute('rel', 'preload');
        link.setAttribute('as', 'style');
        link.setAttribute('onload', "this.rel='stylesheet'");
      }
    });
  }

  /** Parse the stylesheet within a <style> element, then reduce it to contain only rules used by the document. */
  processStyle(style) {
    const done = Promise.resolve();
    const document = style.ownerDocument;

    // basically `.textContent`
    let sheet = style.childNodes.length>0 && style.childNodes.map( node => node.nodeValue ).join('\n');

    // store a reference to the previous serialized stylesheet for reporting stats
    const before = sheet;

    // Skip empty stylesheets
    if (!sheet) return done;

    const ast = css.parse(sheet);

    // Walk all CSS rules, transforming unused rules to comments (which get removed)
    visit(ast, (rule) => {
      if (rule.type==='rule') {
        rule.selectors = rule.selectors.filter((sel) => {
          // Remove unknown pseudos as they break nwmatcher
          sel = sel.replace(/::?(?:[a-z-]+)([.[#~&^:*]|\s|\n|$)/gi, '$1');
          return document.querySelector(sel, document) != null;
        });
        // If there are no matched selectors, replace the rule with a comment.
        if (rule.selectors.length===0) {
          rule.type = 'comment';
          rule.comment = '';
          delete rule.selectors;
          delete rule.declarations;
        }
      }

{'fraction_non_alphanumeric': 0.12945333202473255, 'fraction_numerical': 0.002453626459907744, 'mean_word_length': 2.7881040892193307, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7037644', 'n_tokens_mistral': 2786, 'n_tokens_neox': 2737, 'n_words': 803}