<commit_before>/**
 * Browsertime (http://www.browsertime.com)
 * Copyright (c) 2014, <NAME>, <NAME>
 * and other contributors
 * Released under the Apache 2.0 License
 */
var Stats = require('fast-stats').Stats,
  fs = require('fs'),
  async = require('async'),
  path = require('path'),
  MobProxy = require('./mobproxy'),
  validBrowsers = ['chrome', 'firefox', 'phantomjs'];

function BT() {}

function help() {
  console.log(' -u  the URL to test');
  console.log(' -f  base file name');
  console.log(' -b the browser to use, use one of the following:' + validBrowsers);
  console.log(' -n the number of runs [7]');
  console.log(' --userAgent the user agent String');
  console.log(' -w the screensize wXh');
  console.log(' --scriptPath the path to an alternative script folder');
  console.log(' --headers request headers in the format of ...');
  console.log(' --basicAuth login:password for basic auth');
}

BT.prototype.fetch = function(options, cb) {

  cb = cb || function() {};

  this._checkInput(options);

  var work = [];
  var result = [];
  var self = this;
  var scripts = [];

  // read all the scripts
  var rootPath = path.join(__dirname, 'scripts', path.sep);
  fs.readdirSync(rootPath).forEach(function(file) {
    scripts.push(require(rootPath + file));
  });

  // get scripts from your own configured script dir
  if (options.scriptPath) {
    var p = path.resolve(options.scriptPath);

    fs.readdirSync(p).forEach(function(file) {
      scripts.push(require(p + path.sep + file));
    });
  }

  var mobProxy = new MobProxy({
    port: 10800
  });

  mobProxy.start(function() {

    for (var i = 0; i < options.runs; i++) {
      work.push(function(callback) {
        if (i !== 0) mobProxy.newPage('run' + i);
        var driver = require(path.join(__dirname, 'browsers', path.sep, options.browser)).getDriver(options);

        // fetch the URL and wait until the load event ends or we get the time out
        driver.get(options.url);
        driver.wait(function() {
          return driver.executeScript('return window.performance.timing.loadEventEnd>0').then(function(fetched) {
            return fetched;
          });
        }, 60000);

        // lets run all scripts
        var promises = [];
        scripts.forEach(function(script) {
          promises.push(driver.executeScript(script));
        });

        var callbacks = [];
        promises.forEach(function(promise) {
          callbacks.push(function(cb) {
            promise.then(function(value) {
              cb(null, value);
            });
          });
        });

        // when we are finished, push the result and stop the browser
        async.parallel(callbacks,
          function(err, results) {
            result.push(results);
            driver.quit();
            callback();
          });
      });
    }

    // create statistics for the values we want
    // this should be done cleaner
    async.series(work,
      function(err, results) {
        // fetch timings for each run and make some statistics
        var timings = {};
        result.forEach(function(run) {
          run.forEach(function(metric) {
            self._setupTimingsStatistics(timings, metric);
            self._setupStatistics(timings, metric, 'speedIndex');
            self._setupStatistics(timings, metric, 'firstPaint');
            self._setupUserTimingsStatistics(timings, metric);
          });
        });

        var data = {
          url: options.url,
          browser: options.browser,
          runs: options.times,
          statistics: self._formatStatistics(timings),
          data: result
        };

        // lets store the files
        async.parallel([

            function(callback) {
              fs.writeFile(path.join(__dirname, options.filename + '.json'), JSON.stringify(data), function(err) {
                if (err) throw err;
                callback();
              });
            },
            function(callback) {
              mobProxy.saveHar(path.join(__dirname, options.filename + '.har'), function() {
                mobProxy.stop(callback);
              });
            }<commit_msg>jshints<commit_after>/**
 * Browsertime (http://www.browsertime.com)
 * Copyright (c) 2014, <NAME>, <NAME>
 * and other contributors
 * Released under the Apache 2.0 License
 */
var Stats = require('fast-stats').Stats,
  fs = require('fs'),
  async = require('async'),
  path = require('path'),
  MobProxy = require('./mobproxy'),
  validBrowsers = ['chrome', 'firefox', 'phantomjs'];

function BT() {}

function help() {
  console.log(' -u  the URL to test');
  console.log(' -f  base file name');
  console.log(' -b the browser to use, use one of the following:' + validBrowsers);
  console.log(' -n the number of runs [7]');
  console.log(' --userAgent the user agent String');
  console.log(' -w the screensize wXh');
  console.log(' --scriptPath the path to an alternative script folder');
  console.log(' --headers request headers in the format of ...');
  console.log(' --basicAuth login:password for basic auth');
}

BT.prototype.fetch = function(options, cb) {

  cb = cb || function() {};

  this._checkInput(options);

  var work = [];
  var result = [];
  var self = this;
  var scripts = [];

  // read all the scripts
  var rootPath = path.join(__dirname, 'scripts', path.sep);
  fs.readdirSync(rootPath).forEach(function(file) {
    scripts.push(require(rootPath + file));
  });

  // get scripts from your own configured script dir
  if (options.scriptPath) {
    var p = path.resolve(options.scriptPath);

    fs.readdirSync(p).forEach(function(file) {
      scripts.push(require(p + path.sep + file));
    });
  }

  var mobProxy = new MobProxy({
    port: 10800
  });

  mobProxy.start(function() {

    for (var i = 0; i < options.runs; i++) {
      work.push(function(callback) {
        if (i !== 0) {
          mobProxy.newPage('run' + i);
        }
        var driver = require(path.join(__dirname, 'browsers', path.sep, options.browser)).getDriver(options);

        // fetch the URL and wait until the load event ends or we get the time out
        driver.get(options.url);
        driver.wait(function() {
          return driver.executeScript('return window.performance.timing.loadEventEnd>0').then(function(fetched) {
            return fetched;
          });
        }, 60000);

        // lets run all scripts
        var promises = [];
        scripts.forEach(function(script) {
          promises.push(driver.executeScript(script));
        });

        var callbacks = [];
        promises.forEach(function(promise) {
          callbacks.push(function(cb) {
            promise.then(function(value) {
              cb(null, value);
            });
          });
        });

        // when we are finished, push the result and stop the browser
        async.parallel(callbacks,
          function(err, results) {
            result.push(results);
            driver.quit();
            callback();
          });
      });
    }

    // create statistics for the values we want
    // this should be done cleaner
    async.series(work,
      function(err, results) {
        // fetch timings for each run and make some statistics
        var timings = {};
        result.forEach(function(run) {
          run.forEach(function(metric) {
            self._setupTimingsStatistics(timings, metric);
            self._setupStatistics(timings, metric, 'speedIndex');
            self._setupStatistics(timings, metric, 'firstPaint');
            self._setupUserTimingsStatistics(timings, metric);
          });
        });

        var data = {
          url: options.url,
          browser: options.browser,
          runs: options.times,
          statistics: self._formatStatistics(timings),
          data: result
        };

        // lets store the files
        async.parallel([

            function(callback) {
              fs.writeFile(path.join(__dirname, options.filename + '.json'), JSON.stringify(data), function(err) {
                if (err) {
                  throw err;
                }
                callback();
              });
            },
            function(callback) {
              mobProxy.saveHar(path.join(__dirname, options.filename + '.har'), function() {
                mobProxy.stop(callback);
              });
            }
{'fraction_non_alphanumeric': 0.1364835692012473, 'fraction_numerical': 0.004797313504437515, 'mean_word_length': 2.203611217825586, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7105476', 'n_tokens_mistral': 2374, 'n_tokens_neox': 2297, 'n_words': 598}