<commit_before>/**
 * Copyright 2018 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

import path from 'path';
import WORKER_PLUGIN_SYMBOL from './symbol';
let ParserHelpers;
try {
  ParserHelpers = require('webpack/lib/javascript/JavascriptParserHelpers'); // Webpack 5
} catch (e) {}
ParserHelpers = ParserHelpers || require('webpack/lib/ParserHelpers'); // Webpack 4
let HarmonyImportSpecifierDependency;
try {
  HarmonyImportSpecifierDependency = require('webpack/lib/dependencies/HarmonyImportSpecifierDependency');
} catch (e) {}

const NAME = 'WorkerPlugin';
const workerLoader = path.resolve(__dirname, 'loader.js');

export default class WorkerPlugin {
  constructor (options) {
    this.options = options || {};
    this[WORKER_PLUGIN_SYMBOL] = true;
  }

  apply (compiler) {
    compiler.hooks.normalModuleFactory.tap(NAME, factory => {
      let workerId = 0;
      factory.hooks.parser.for('javascript/auto').tap(NAME, parser => parse(parser, false));
      factory.hooks.parser.for('javascript/dynamic').tap(NAME, parser => parse(parser, false));
      factory.hooks.parser.for('javascript/esm').tap(NAME, parser => parse(parser, true));

      const parse = (parser, esModule) => {
        const handleWorker = workerTypeString => expr => {
          const dep = parser.evaluateExpression(expr.arguments[0]);

          const optsExpr = expr.arguments[1];
          let hasInitOptions = false;
          let typeModuleExpr;
          let opts;
          if (optsExpr) {
            opts = {};
            for (let i = optsExpr.properties.length; i--;) {
              const prop = optsExpr.properties[i];
              if (prop.type === 'Property' && !prop.computed && !prop.shorthand && !prop.method) {
                opts[prop.key.name] = parser.evaluateExpression(prop.value).string;

                if (prop.key.name === 'type') {
                  typeModuleExpr = prop;
                } else {
                  hasInitOptions = true;
                }
              }
            }
          }

          if (!opts || opts.type !== 'module') {
            // If an unknown type value is passed, it's probably an error and we can warn the developer:
            if (opts && opts.type !== 'classic') {
              parser.state.module.warnings.push({
                message: `new ${workerTypeString}() will only be bundled if passed options that include { type: 'module' }.${opts ? `\n  Received: new ${workerTypeString}()(${JSON.stringify(dep.string)}, ${JSON.stringify(opts)})` : ''}`
              });
            }
            return false;
          }

          if (!dep.isString()) {<commit_msg>Fix `Module parse failed: Cannot read property 'length' of undefined`<commit_after>/**
 * Copyright 2018 Google LLC
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
 * use this file except in compliance with the License. You may obtain a copy of
 * the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 * License for the specific language governing permissions and limitations under
 * the License.
 */

import path from 'path';
import WORKER_PLUGIN_SYMBOL from './symbol';
let ParserHelpers;
try {
  ParserHelpers = require('webpack/lib/javascript/JavascriptParserHelpers'); // Webpack 5
} catch (e) {}
ParserHelpers = ParserHelpers || require('webpack/lib/ParserHelpers'); // Webpack 4
let HarmonyImportSpecifierDependency;
try {
  HarmonyImportSpecifierDependency = require('webpack/lib/dependencies/HarmonyImportSpecifierDependency');
} catch (e) {}

const NAME = 'WorkerPlugin';
const workerLoader = path.resolve(__dirname, 'loader.js');

export default class WorkerPlugin {
  constructor (options) {
    this.options = options || {};
    this[WORKER_PLUGIN_SYMBOL] = true;
  }

  apply (compiler) {
    compiler.hooks.normalModuleFactory.tap(NAME, factory => {
      let workerId = 0;
      factory.hooks.parser.for('javascript/auto').tap(NAME, parser => parse(parser, false));
      factory.hooks.parser.for('javascript/dynamic').tap(NAME, parser => parse(parser, false));
      factory.hooks.parser.for('javascript/esm').tap(NAME, parser => parse(parser, true));

      const parse = (parser, esModule) => {
        const handleWorker = workerTypeString => expr => {
          const dep = parser.evaluateExpression(expr.arguments[0]);

          const optsExpr = expr.arguments[1];
          let hasInitOptions = false;
          let typeModuleExpr;
          let opts;
          if (optsExpr && optsExpr.properties) {
            opts = {};
            for (let i = optsExpr.properties.length; i--;) {
              const prop = optsExpr.properties[i];
              if (prop.type === 'Property' && !prop.computed && !prop.shorthand && !prop.method) {
                opts[prop.key.name] = parser.evaluateExpression(prop.value).string;

                if (prop.key.name === 'type') {
                  typeModuleExpr = prop;
                } else {
                  hasInitOptions = true;
                }
              }
            }
          }

          if (!opts || opts.type !== 'module') {
            // If an unknown type value is passed, it's probably an error and we can warn the developer:
            if (opts && opts.type !== 'classic') {
              parser.state.module.warnings.push({
                message: `new ${workerTypeString}() will only be bundled if passed options that include { type: 'module' }.${opts ? `\n  Received: new ${workerTypeString}()(${JSON.stringify(dep.string)}, ${JSON.stringify(opts)})` : ''}`
              });
            }
            return false;
          }

          if (!dep.isString()) {
{'fraction_non_alphanumeric': 0.1240407204385278, 'fraction_numerical': 0.0040720438527799534, 'mean_word_length': 2.6263486655309483, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '172189', 'n_tokens_mistral': 1745, 'n_tokens_neox': 1695, 'n_words': 503}