<commit_before>'use strict';


const selenium = require('selenium-standalone');
const cp = require('child_process');
const path = require('path');
const fs = require('fs');
const mkdirp = require('mkdirp');
const util = require('util');


const NIGHTWATCH_CONFIG = process.env.NIGHTWATCH_CONFIG;
const LOG_DIR = process.env.LOG_DIR;
const REPORT_DIR = process.env.REPORT_DIR;
const NIGHTWATCH_ENV = process.env.NIGHTWATCH_ENV;
const WEBPACK_CONFIG = process.env.WEBPACK_CONFIG;
const NODE_PORT = process.env.NODE_PORT;
const PORT = process.env.PORT;
const TUNNEL_IDENTIFIER = process.env.TUNNEL_IDENTIFIER;


const createWebpackServer = config => {
  const webpack = require('webpack');
  const WebpackDevServer = require('webpack-dev-server');

  const webpackConfig = require(config.webpackConfig);

  return new WebpackDevServer(webpack(webpackConfig), {quiet: true});
};


const getConfig = options => {
  let nightwatchConfig = NIGHTWATCH_CONFIG ?
    path.resolve(NIGHTWATCH_CONFIG) : path.resolve(process.cwd(), 'nightwatch.json');

  try {
    require(nightwatchConfig);
  } catch (err) {
    console.log('Fallback to default nightwatch.json config');
    nightwatchConfig = path.resolve(__dirname, 'nightwatch.json');
  }

  const config = Object.assign({
    logDir: LOG_DIR ?
      path.resolve(LOG_DIR) : path.resolve(process.cwd(), 'reports'),
    reportDir: REPORT_DIR ?
      path.resolve(REPORT_DIR) : path.resolve(process.cwd(), 'reports', 'test-e2e'),
    nightwatchConfig,
    nightwatchEnv: NIGHTWATCH_ENV || 'default',
    webpackConfig: WEBPACK_CONFIG ?
      path.resolve(WEBPACK_CONFIG) : path.resolve(process.cwd(), 'webpack.config.js'),
    port: NODE_PORT || PORT || 8080,
    tunnelIdentifier: TUNNEL_IDENTIFIER || undefined
  }, options);

  console.log(`Running with config:\n${util.inspect(config, {depth: 0, colors: true})}`);

  return Object.assign({}, config, {
    server: config.server || createWebpackServer(config)
  });
};


module.exports = options => {
  const config = getConfig(options);

  const startServer = callback =>
    config.server.listen(config.port, '0.0.0.0', callback);


  const onServerStarted = seleniumChild => err => {
    if (err) {
      console.error(err.stack);
      process.exit(1);
    }

    mkdirp.sync(config.reportDir);

    const which = require('npm-which')(__dirname);

    const nightwatchRunner = which.sync('nightwatch');

    const args = [
      '--config', config.nightwatchConfig,
      '--env', config.nightwatchEnv,
      '--output', config.reportDir
    ];

    cp.spawn(nightwatchRunner, args, {stdio: 'inherit'})
      .on('error', err2 => {
        console.error(err2.stack);
        process.exit(1);
      })
      .on('close', code => {
        seleniumChild.kill('SIGINT');
        process.exit(code);
      });
  };


  function onSeleniumStarted(err, seleniumChild) {
    if (err) {
      console.error(err.stack);
      process.exit(1);
    }

    mkdirp.sync(config.logDir);

    const seleniumLog = fs.createWriteStream(path.resolve(config.logDir, 'selenium.log'));

    seleniumChild.stdout.pipe(seleniumLog);
    seleniumChild.stderr.pipe(seleniumLog);

    process.on('uncaughtException', err2 => {
      console.error(err2.stack);
      seleniumChild.kill('SIGINT');
      process.exit(1);
    });

    startServer(onServerStarted(seleniumChild));
  }


  function onSeleniumInstalled(err) {
    if (err) {
      console.error(err.stack);
      process.exit(1);
    }

    selenium.start({seleniumArgs: ['-debug']}, onSeleniumStarted);
  }

  let progress;
  selenium.install({
    logger: message => console.log(message),
    progressCb: (totalLength, progressLength) => {
      const newProgress = (100 * (progressLength / totalLength));
      if (!progress || (newProgress - progress > 5) || progress > 99.99) {
        progress = newProgress;
        console.log(`Downloading: ${progress.toFixed(2)}%`);
      }
    }
  }, onSeleniumInstalled);
};<commit_msg>More logging for 99%<commit_after>'use strict';


const selenium = require('selenium-standalone');
const cp = require('child_process');
const path = require('path');
const fs = require('fs');
const mkdirp = require('mkdirp');
const util = require('util');


const NIGHTWATCH_CONFIG = process.env.NIGHTWATCH_CONFIG;
const LOG_DIR = process.env.LOG_DIR;
const REPORT_DIR = process.env.REPORT_DIR;
const NIGHTWATCH_ENV = process.env.NIGHTWATCH_ENV;
const WEBPACK_CONFIG = process.env.WEBPACK_CONFIG;
const NODE_PORT = process.env.NODE_PORT;
const PORT = process.env.PORT;
const TUNNEL_IDENTIFIER = process.env.TUNNEL_IDENTIFIER;


const createWebpackServer = config => {
  const webpack = require('webpack');
  const WebpackDevServer = require('webpack-dev-server');

  const webpackConfig = require(config.webpackConfig);

  return new WebpackDevServer(webpack(webpackConfig), {quiet: true});
};


const getConfig = options => {
  let nightwatchConfig = NIGHTWATCH_CONFIG ?
    path.resolve(NIGHTWATCH_CONFIG) : path.resolve(process.cwd(), 'nightwatch.json');

  try {
    require(nightwatchConfig);
  } catch (err) {
    console.log('Fallback to default nightwatch.json config');
    nightwatchConfig = path.resolve(__dirname, 'nightwatch.json');
  }

  const config = Object.assign({
    logDir: LOG_DIR ?
      path.resolve(LOG_DIR) : path.resolve(process.cwd(), 'reports'),
    reportDir: REPORT_DIR ?
      path.resolve(REPORT_DIR) : path.resolve(process.cwd(), 'reports', 'test-e2e'),
    nightwatchConfig,
    nightwatchEnv: NIGHTWATCH_ENV || 'default',
    webpackConfig: WEBPACK_CONFIG ?
      path.resolve(WEBPACK_CONFIG) : path.resolve(process.cwd(), 'webpack.config.js'),
    port: NODE_PORT || PORT || 8080,
    tunnelIdentifier: TUNNEL_IDENTIFIER || undefined
  }, options);

  console.log(`Running with config:\n${util.inspect(config, {depth: 0, colors: true})}`);

  return Object.assign({}, config, {
    server: config.server || createWebpackServer(config)
  });
};


module.exports = options => {
  const config = getConfig(options);

  const startServer = callback =>
    config.server.listen(config.port, '0.0.0.0', callback);


  const onServerStarted = seleniumChild => err => {
    if (err) {
      console.error(err.stack);
      process.exit(1);
    }

    mkdirp.sync(config.reportDir);

    const which = require('npm-which')(__dirname);

    const nightwatchRunner = which.sync('nightwatch');

    const args = [
      '--config', config.nightwatchConfig,
      '--env', config.nightwatchEnv,
      '--output', config.reportDir
    ];

    cp.spawn(nightwatchRunner, args, {stdio: 'inherit'})
      .on('error', err2 => {
        console.error(err2.stack);
        process.exit(1);
      })
      .on('close', code => {
        seleniumChild.kill('SIGINT');
        process.exit(code);
      });
  };


  function onSeleniumStarted(err, seleniumChild) {
    if (err) {
      console.error(err.stack);
      process.exit(1);
    }

    mkdirp.sync(config.logDir);

    const seleniumLog = fs.createWriteStream(path.resolve(config.logDir, 'selenium.log'));

    seleniumChild.stdout.pipe(seleniumLog);
    seleniumChild.stderr.pipe(seleniumLog);

    process.on('uncaughtException', err2 => {
      console.error(err2.stack);
      seleniumChild.kill('SIGINT');
      process.exit(1);
    });

    startServer(onServerStarted(seleniumChild));
  }


  function onSeleniumInstalled(err) {
    if (err) {
      console.error(err.stack);
      process.exit(1);
    }

    selenium.start({seleniumArgs: ['-debug']}, onSeleniumStarted);
  }

  let progress;
  selenium.install({
    logger: message => console.log(message),
    progressCb: (totalLength, progressLength) => {
      const newProgress = (100 * (progressLength / totalLength));
      if (!progress || (newProgress - progress > 5) || progress > 99) {
        progress = newProgress;
        console.log(`Downloading: ${progress.toFixed(2)}%`);
      }
    }
  }, onSeleniumInstalled);
};
{'fraction_non_alphanumeric': 0.1469994957135653, 'fraction_numerical': 0.0070600100857286935, 'mean_word_length': 3.7080118694362016, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '438957', 'n_tokens_mistral': 2630, 'n_tokens_neox': 2514, 'n_words': 457}