<commit_before>const childProcess = require('child_process')
const fs = require('fs')
const path = require('path')
const electronLink = require('electron-link')
const CONFIG = require('../config')
const vm = require('vm')

module.exports = function (packagedAppPath) {
  const snapshotScriptPath = path.join(CONFIG.buildOutputPath, 'startup.js')
  const coreModules = new Set(['electron', 'atom', 'shell', 'WNdb', 'lapack', 'remote'])
  const baseDirPath = path.join(CONFIG.intermediateAppPath, 'static')
  let processedFiles = 0

  return electronLink({
    baseDirPath,
    mainPath: path.resolve(baseDirPath, '..', 'src', 'initialize-application-window.js'),
    cachePath: path.join(CONFIG.atomHomeDirPath, 'snapshot-cache'),
    auxiliaryData: CONFIG.snapshotAuxiliaryData,
    shouldExcludeModule: (modulePath) => {
      if (processedFiles > 0) {
        process.stdout.write('\r')
      }
      process.stdout.write(`Generating snapshot script at "${snapshotScriptPath}" (${++processedFiles})`)

      const relativePath = path.relative(baseDirPath, modulePath)
      return (
        modulePath.endsWith('.node') ||
        coreModules.has(modulePath) ||
        (relativePath.startsWith(path.join('..', 'src')) && relativePath.endsWith('-element.js')) ||
        relativePath.startsWith(path.join('..', 'node_modules', 'dugite')) ||
        relativePath == path.join('..', 'exports', 'atom.js') ||
        relativePath == path.join('..', 'src', 'electron-shims.js') ||
        relativePath == path.join('..', 'src', 'safe-clipboard.js') ||
        relativePath == path.join('..', 'node_modules', 'atom-keymap', 'lib', 'command-event.js') ||
        relativePath == path.join('..', 'node_modules', 'babel-core', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'cached-run-in-this-context', 'lib', 'main.js') ||
        relativePath == path.join('..', 'node_modules', 'coffee-script', 'lib', 'coffee-script', 'register.js') ||
        relativePath == path.join('..', 'node_modules', 'cson-parser', 'node_modules', 'coffee-script', 'lib', 'coffee-script', 'register.js') ||
        relativePath == path.join('..', 'node_modules', 'decompress-zip', 'lib', 'decompress-zip.js') ||
        relativePath == path.join('..', 'node_modules', 'debug', 'node.js') ||
        relativePath == path.join('..', 'node_modules', 'fs-extra', 'lib', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'git-utils', 'lib', 'git.js') ||
        relativePath == path.join('..', 'node_modules', 'glob', 'glob.js') ||
        relativePath == path.join('..', 'node_modules', 'graceful-fs', 'graceful-fs.js') ||
        relativePath == path.join('..', 'node_modules', 'htmlparser2', 'lib', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'iconv-lite', 'encodings', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'less', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'less', 'lib', 'less', 'fs.js') ||
        relativePath == path.join('..', 'node_modules', 'less', 'lib', 'less-node', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'less', 'node_modules', 'graceful-fs', 'graceful-fs.js') ||
        relativePath == path.join('..', 'node_modules', 'minimatch', 'minimatch.js') ||
        relativePath == path.join('..', 'node_modules', 'node-fetch', 'lib', 'fetch-error.js') ||
        relativePath == path.join('..', 'node_modules', 'nsfw', 'node_modules', 'fs-extra', 'lib', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'superstring', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'oniguruma', 'src', 'oniguruma.js') ||
        relativePath == path.join('..', 'node_modules', 'request', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'resolve', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'resolve', 'lib', 'core.js') ||
        relativePath == path.join('..', 'node_modules', 'scandal', 'node_modules', 'minimatch', 'minimatch.js') ||
        relativePath == path.join('..', 'node_modules', 'settings-view', 'node_modules', 'glob', 'glob.js') ||
        relativePath == path.join('..', 'node_modules', 'settings-view', 'node_modules', 'minimatch', 'minimatch.js') ||
        relativePath == path.join('..', 'node_modules', 'spellchecker', 'lib', 'spellchecker.js') ||
        relativePath == path.join('..', 'node_modules', 'spelling-manager', 'node_modules', 'natural', 'lib', 'natural', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'tar', 'tar.js') ||
        relativePath == path.join('..', 'node_modules', 'temp', 'lib', 'temp.js') ||
        relativePath == path.join('..', 'node_modules', 'tmp', 'lib', 'tmp.js') ||
        relativePath == path.join('..', 'node_modules', 'tree-view', 'node_modules', 'minimatch', 'minimatch.js')<commit_msg>Handle more htmlparser2 snapshot exceptions<commit_after>const childProcess = require('child_process')
const fs = require('fs')
const path = require('path')
const electronLink = require('electron-link')
const CONFIG = require('../config')
const vm = require('vm')

module.exports = function (packagedAppPath) {
  const snapshotScriptPath = path.join(CONFIG.buildOutputPath, 'startup.js')
  const coreModules = new Set(['electron', 'atom', 'shell', 'WNdb', 'lapack', 'remote'])
  const baseDirPath = path.join(CONFIG.intermediateAppPath, 'static')
  let processedFiles = 0

  return electronLink({
    baseDirPath,
    mainPath: path.resolve(baseDirPath, '..', 'src', 'initialize-application-window.js'),
    cachePath: path.join(CONFIG.atomHomeDirPath, 'snapshot-cache'),
    auxiliaryData: CONFIG.snapshotAuxiliaryData,
    shouldExcludeModule: (modulePath) => {
      if (processedFiles > 0) {
        process.stdout.write('\r')
      }
      process.stdout.write(`Generating snapshot script at "${snapshotScriptPath}" (${++processedFiles})`)

      const relativePath = path.relative(baseDirPath, modulePath)
      return (
        modulePath.endsWith('.node') ||
        coreModules.has(modulePath) ||
        (relativePath.startsWith(path.join('..', 'src')) && relativePath.endsWith('-element.js')) ||
        relativePath.startsWith(path.join('..', 'node_modules', 'dugite')) ||
        relativePath == path.join('..', 'exports', 'atom.js') ||
        relativePath == path.join('..', 'src', 'electron-shims.js') ||
        relativePath == path.join('..', 'src', 'safe-clipboard.js') ||
        relativePath == path.join('..', 'node_modules', 'atom-keymap', 'lib', 'command-event.js') ||
        relativePath == path.join('..', 'node_modules', 'babel-core', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'cached-run-in-this-context', 'lib', 'main.js') ||
        relativePath == path.join('..', 'node_modules', 'coffee-script', 'lib', 'coffee-script', 'register.js') ||
        relativePath == path.join('..', 'node_modules', 'cson-parser', 'node_modules', 'coffee-script', 'lib', 'coffee-script', 'register.js') ||
        relativePath == path.join('..', 'node_modules', 'decompress-zip', 'lib', 'decompress-zip.js') ||
        relativePath == path.join('..', 'node_modules', 'debug', 'node.js') ||
        relativePath == path.join('..', 'node_modules', 'fs-extra', 'lib', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'git-utils', 'lib', 'git.js') ||
        relativePath == path.join('..', 'node_modules', 'glob', 'glob.js') ||
        relativePath == path.join('..', 'node_modules', 'graceful-fs', 'graceful-fs.js') ||
        relativePath == path.join('..', 'node_modules', 'htmlparser2', 'lib', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'markdown-preview', 'node_modules', 'htmlparser2', 'lib', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'roaster', 'node_modules', 'htmlparser2', 'lib', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'task-lists', 'node_modules', 'htmlparser2', 'lib', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'iconv-lite', 'encodings', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'less', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'less', 'lib', 'less', 'fs.js') ||
        relativePath == path.join('..', 'node_modules', 'less', 'lib', 'less-node', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'less', 'node_modules', 'graceful-fs', 'graceful-fs.js') ||
        relativePath == path.join('..', 'node_modules', 'minimatch', 'minimatch.js') ||
        relativePath == path.join('..', 'node_modules', 'node-fetch', 'lib', 'fetch-error.js') ||
        relativePath == path.join('..', 'node_modules', 'nsfw', 'node_modules', 'fs-extra', 'lib', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'superstring', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'oniguruma', 'src', 'oniguruma.js') ||
        relativePath == path.join('..', 'node_modules', 'request', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'resolve', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'resolve', 'lib', 'core.js') ||
        relativePath == path.join('..', 'node_modules', 'scandal', 'node_modules', 'minimatch', 'minimatch.js') ||
        relativePath == path.join('..', 'node_modules', 'settings-view', 'node_modules', 'glob', 'glob.js') ||
        relativePath == path.join('..', 'node_modules', 'settings-view', 'node_modules', 'minimatch', 'minimatch.js') ||
        relativePath == path.join('..', 'node_modules', 'spellchecker', 'lib', 'spellchecker.js') ||
        relativePath == path.join('..', 'node_modules', 'spelling-manager', 'node_modules', 'natural', 'lib', 'natural', 'index.js') ||
        relativePath == path.join('..', 'node_modules', 'tar', 'tar.js') ||
        relativePath == path.join('..', 'node_modules', 'temp', 'lib', 'temp.js') ||
        relativePath == path.join('..', 'node_modules', 'tmp', 'lib', 'tmp.js') ||
        relativePath == path.join('..', 'node_modules', 'tree-view', 'node_modules', 'minimatch', 'minimatch.js')
{'fraction_non_alphanumeric': 0.2311764705882353, 'fraction_numerical': 0.000980392156862745, 'mean_word_length': 3.850689491203043, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6526528', 'n_tokens_mistral': 3259, 'n_tokens_neox': 3247, 'n_words': 627}