<commit_before>/**
 * Copyright 2015 The AMP HTML Authors. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS-IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
'use strict';
const argv = require('minimist')(process.argv.slice(2));
const del = require('del');
const fs = require('fs-extra');
const gulp = require('gulp');
const gulpIf = require('gulp-if');
const nop = require('gulp-nop');
const pathModule = require('path');
const rename = require('gulp-rename');
const sourcemaps = require('gulp-sourcemaps');
const {
  gulpClosureCompile,
  handleCompilerError,
  handleTypeCheckError,
} = require('./closure-compile');
const {checkForUnknownDeps} = require('./check-for-unknown-deps');
const {CLOSURE_SRC_GLOBS} = require('./sources');
const {cpus} = require('os');
const {green, cyan} = require('kleur/colors');
const {log, logLocalDev} = require('../common/logging');
const {postClosureBabel} = require('./post-closure-babel');
const {preClosureBabel, handlePreClosureError} = require('./pre-closure-babel');
const {sanitize} = require('./sanitize');
const {VERSION: internalRuntimeVersion} = require('./internal-version');
const {writeSourcemaps} = require('./helpers');

const queue = [];
let inProgress = 0;

const MAX_PARALLEL_CLOSURE_INVOCATIONS =
  parseInt(argv.closure_concurrency, 10) || cpus().length;

/**
 * Compiles AMP with the closure compiler. This is intended only for
 * production use. During development we intend to continue using
 * babel, as it has much faster incremental compilation.
 *
 * @param {string} entryModuleFilename
 * @param {string} outputDir
 * @param {string} outputFilename
 * @param {{
 *  esmPassCompilation?: string,
 *  wrapper?: string,
 *  extraGlobs?: string,
 *  include3pDirectories?: boolean,
 *  includePolyfills?: boolean,
 *  externs?: string[],
 *  compilationLevel?: string,
 *  verboseLogging?: boolean,
 *  typeCheckOnly?: boolean,
 *  skipUnknownDepsCheck?: boolean,
 * }} options
 * @param {{startTime?: number}} timeInfo
 * @return {Promise<void>}
 */
async function closureCompile(
  entryModuleFilename,
  outputDir,
  outputFilename,
  options,
  timeInfo
) {
  // Rate limit closure compilation to MAX_PARALLEL_CLOSURE_INVOCATIONS
  // concurrent processes.
  return new Promise(function (resolve, reject) {
    /**
     * @return {void}
     */
    function start() {
      inProgress++;
      compile(
        entryModuleFilename,
        outputDir,
        outputFilename,
        options,
        timeInfo
      ).then(
        function () {
          inProgress--;
          next();
          resolve();
        },
        (reason) => reject(reason)
      );
    }
    /**
     * @return {void}
     */
    function next() {
      if (!queue.length) {
        return;
      }
      if (inProgress < MAX_PARALLEL_CLOSURE_INVOCATIONS) {
        queue.shift()();
      }
    }
    queue.push(start);
    next();
  });
}

/**
 * @return {void}
 */
function cleanupBuildDir() {
  del.sync('build/fake-module');
  del.sync('build/patched-module');
  del.sync('build/parsers');
  fs.mkdirsSync('build/fake-module/third_party/babel');
  fs.mkdirsSync('build/fake-module/src/polyfills/');
  fs.mkdirsSync('build/fake-polyfills/src/polyfills');
}

/**
 * @param {string[]|string} entryModuleFilenames
 * @param {string} outputDir
 * @param {string} outputFilename
 * @param {{
 *  esmPassCompilation?: string,
 *  wrapper?: string,
 *  extraGlobs?: string,
 *  include3pDirectories?: boolean,
 *  includePolyfills?: boolean,
 *  externs?: string[],
 *  compilationLevel?: string,
 *  verboseLogging?: boolean,
 *  typeCheckOnly?: boolean,
 *  skipUnknownDepsCheck?: boolean,
 * }} options
 * @param {{startTime?: number}} timeInfo
 * @return {Promise<void>}
 */
function compile(
  entryModuleFilenames,
  outputDir,
  outputFilename,
  options,
  timeInfo
) {
  const hideWarningsFor = [
    'third_party/amp-toolbox-cache-url/',
    'third_party/caja/',
    'third_party/closure-library/sha384-generated.js',
    'third_party/d3/',
    'third_party/inputmask/',
    'third_party/mustache/',
    'third_party/react-dates/',
    'third_party/set-dom/',
    'third_party/subscriptions-project/',
    'third_party/vega/',
    'third_party/webcomponentsjs/',
    'node_modules/',
    'build/patched-module/',
  ];
  const baseExterns = [
    'build-system/externs/amp.extern.js',
    'build-system/externs/dompurify.extern.js',
    'build-system/externs/layout-jank.extern.js',
    'build-system/externs/performance-observer.extern.js',
    'third_party/web-animations-externs/web_animations.js',
    'third_party/moment/moment.extern.js',
    'third_party/react-externs/externs.js',
    'build-system/externs/preact.extern.js',
    'build-system/externs/weakref.extern.js',
  ];
  const define = [`VERSION=${internalRuntimeVersion}`];
  if (argv.pseudo_names) {
    define.push('PSEUDO_NAMES=true');
  }

  return new Promise(function (resolve, reject) {
    if (!(entryModuleFilenames instanceof Array)) {
      entryModuleFilenames = [entryModuleFilenames];
    }
    const unneededFiles = [
      'build/fake-module/third_party/babel/custom-babel-helpers.js',
    ];
    let wrapper = '(function(){%output%})();';
    if (options.wrapper) {
      wrapper = options.wrapper.replace('<%= contents %>', '%output%');
    }
    const srcs = [...CLOSURE_SRC_GLOBS];
    // Add needed path for extensions.
    // Instead of globbing all extensions, this will only add the actual
    // extension path for much quicker build times.
    entryModuleFilenames.forEach(function (filename) {
      if (!pathModule.normalize(filename).startsWith('extensions')) {
        return;
      }<commit_msg>Include AMP_MODE when closure compiling (#33149)<commit_after>/**
 * Copyright 2015 The AMP HTML Authors. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS-IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
'use strict';
const argv = require('minimist')(process.argv.slice(2));
const del = require('del');
const fs = require('fs-extra');
const gulp = require('gulp');
const gulpIf = require('gulp-if');
const nop = require('gulp-nop');
const pathModule = require('path');
const rename = require('gulp-rename');
const sourcemaps = require('gulp-sourcemaps');
const {
  gulpClosureCompile,
  handleCompilerError,
  handleTypeCheckError,
} = require('./closure-compile');
const {checkForUnknownDeps} = require('./check-for-unknown-deps');
const {CLOSURE_SRC_GLOBS} = require('./sources');
const {cpus} = require('os');
const {green, cyan} = require('kleur/colors');
const {log, logLocalDev} = require('../common/logging');
const {postClosureBabel} = require('./post-closure-babel');
const {preClosureBabel, handlePreClosureError} = require('./pre-closure-babel');
const {sanitize} = require('./sanitize');
const {VERSION: internalRuntimeVersion} = require('./internal-version');
const {writeSourcemaps} = require('./helpers');

const queue = [];
let inProgress = 0;

const MAX_PARALLEL_CLOSURE_INVOCATIONS =
  parseInt(argv.closure_concurrency, 10) || cpus().length;

/**
 * Compiles AMP with the closure compiler. This is intended only for
 * production use. During development we intend to continue using
 * babel, as it has much faster incremental compilation.
 *
 * @param {string} entryModuleFilename
 * @param {string} outputDir
 * @param {string} outputFilename
 * @param {{
 *  esmPassCompilation?: string,
 *  wrapper?: string,
 *  extraGlobs?: string,
 *  include3pDirectories?: boolean,
 *  includePolyfills?: boolean,
 *  externs?: string[],
 *  compilationLevel?: string,
 *  verboseLogging?: boolean,
 *  typeCheckOnly?: boolean,
 *  skipUnknownDepsCheck?: boolean,
 * }} options
 * @param {{startTime?: number}} timeInfo
 * @return {Promise<void>}
 */
async function closureCompile(
  entryModuleFilename,
  outputDir,
  outputFilename,
  options,
  timeInfo
) {
  // Rate limit closure compilation to MAX_PARALLEL_CLOSURE_INVOCATIONS
  // concurrent processes.
  return new Promise(function (resolve, reject) {
    /**
     * @return {void}
     */
    function start() {
      inProgress++;
      compile(
        entryModuleFilename,
        outputDir,
        outputFilename,
        options,
        timeInfo
      ).then(
        function () {
          inProgress--;
          next();
          resolve();
        },
        (reason) => reject(reason)
      );
    }
    /**
     * @return {void}
     */
    function next() {
      if (!queue.length) {
        return;
      }
      if (inProgress < MAX_PARALLEL_CLOSURE_INVOCATIONS) {
        queue.shift()();
      }
    }
    queue.push(start);
    next();
  });
}

/**
 * @return {void}
 */
function cleanupBuildDir() {
  del.sync('build/fake-module');
  del.sync('build/patched-module');
  del.sync('build/parsers');
  fs.mkdirsSync('build/fake-module/third_party/babel');
  fs.mkdirsSync('build/fake-module/src/polyfills/');
  fs.mkdirsSync('build/fake-polyfills/src/polyfills');
}

/**
 * @param {string[]|string} entryModuleFilenames
 * @param {string} outputDir
 * @param {string} outputFilename
 * @param {{
 *  esmPassCompilation?: string,
 *  wrapper?: string,
 *  extraGlobs?: string,
 *  include3pDirectories?: boolean,
 *  includePolyfills?: boolean,
 *  externs?: string[],
 *  compilationLevel?: string,
 *  verboseLogging?: boolean,
 *  typeCheckOnly?: boolean,
 *  skipUnknownDepsCheck?: boolean,
 * }} options
 * @param {{startTime?: number}} timeInfo
 * @return {Promise<void>}
 */
function compile(
  entryModuleFilenames,
  outputDir,
  outputFilename,
  options,
  timeInfo
) {
  const hideWarningsFor = [
    'third_party/amp-toolbox-cache-url/',
    'third_party/caja/',
    'third_party/closure-library/sha384-generated.js',
    'third_party/d3/',
    'third_party/inputmask/',
    'third_party/mustache/',
    'third_party/react-dates/',
    'third_party/set-dom/',
    'third_party/subscriptions-project/',
    'third_party/vega/',
    'third_party/webcomponentsjs/',
    'node_modules/',
    'build/patched-module/',
  ];
  const baseExterns = [
    'build-system/externs/amp.extern.js',
    'build-system/externs/dompurify.extern.js',
    'build-system/externs/layout-jank.extern.js',
    'build-system/externs/performance-observer.extern.js',
    'third_party/web-animations-externs/web_animations.js',
    'third_party/moment/moment.extern.js',
    'third_party/react-externs/externs.js',
    'build-system/externs/preact.extern.js',
    'build-system/externs/weakref.extern.js',
  ];
  const define = [`VERSION=${internalRuntimeVersion}`, 'AMP_MODE=true'];
  if (argv.pseudo_names) {
    define.push('PSEUDO_NAMES=true');
  }

  return new Promise(function (resolve, reject) {
    if (!(entryModuleFilenames instanceof Array)) {
      entryModuleFilenames = [entryModuleFilenames];
    }
    const unneededFiles = [
      'build/fake-module/third_party/babel/custom-babel-helpers.js',
    ];
    let wrapper = '(function(){%output%})();';
    if (options.wrapper) {
      wrapper = options.wrapper.replace('<%= contents %>', '%output%');
    }
    const srcs = [...CLOSURE_SRC_GLOBS];
    // Add needed path for extensions.
    // Instead of globbing all extensions, this will only add the actual
    // extension path for much quicker build times.
    entryModuleFilenames.forEach(function (filename) {
      if (!pathModule.normalize(filename).startsWith('extensions')) {
        return;
      }
{'fraction_non_alphanumeric': 0.14477102955505033, 'fraction_numerical': 0.0033290029230269567, 'mean_word_length': 3.6531922931620704, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7126094', 'n_tokens_mistral': 3916, 'n_tokens_neox': 3838, 'n_words': 850}