<commit_before>/**
 * Browsertime (http://www.browsertime.com)
 * Copyright (c) 2014, <NAME>, <NAME>
 * and other contributors
 * Released under the Apache 2.0 License
 */
var stats = require('./statistics'),
  fs = require('fs'),
  async = require('async'),
  path = require('path'),
  MobProxy = require('./mobproxy'),
  log = require('winston'),
  url = require('url');

function Browsertime() {
  this.result = [];
  this.scripts = [];
  this.browserVersion = '';
  this.os = '';
  this.browserName = '';
  this.userAgent = '';
}

Browsertime.prototype.fetch = function(options, cb) {

  cb = cb || function() {};

  this._setupDefaults(options);
  this._populateScripts(options);
  this._setupLogging(options);

  if (options.useProxy === true) {
    // setup the mob proxy
    var mobProxy = new MobProxy({
      port: options.mobProxyPort, // make configurable
      proxyPort: options.proxyPort,
      domain: url.parse(options.url).hostname,
      headers: options.headers,
      basicAuth: options.basicAuth,
      limit: options.connectionRaw,
      proxySleepBeforeStart: options.proxySleepBeforeStart
    });

    log.log('info', 'Start the proxy [' + options.mobProxyPort + '], will wait ' + options.proxySleepBeforeStart +
      ' ms');
    var self = this;
    mobProxy.start(function() {
      self._do(mobProxy, options, cb);
    });
  }
  // if we don't use the proxy
  else {
    this._do(undefined, options, cb);
  }
};

Browsertime.prototype._do = function(mobProxy, options, cb) {
  var browserRuns = [];
  var self = this;
  for (var i = 0; i < options.runs; i++) {
    browserRuns.push(self._collectFromTheBrowser(mobProxy, options));
  }

  async.series(browserRuns,
    function(err, results) {
      var data = self._getFormattedResult(options);
      self._saveToFile(data, options, mobProxy, cb);
    });
};

Browsertime.prototype._collectFromTheBrowser = function(mobProxy, options) {
  var self = this;

  return function(callback) {
    async.series([

        function(callback) {
          if (options.useProxy === true) {
            mobProxy.newPage('myname', callback);
          } else {
            callback();
          }
        },
        function(callback) {
          if (options.useProxy === true) {
            mobProxy.clearDNS(callback);
          } else {
            callback();
          }
        }
      ],
      function(err, results) {
        var driver = require(path.join(__dirname, 'browsers', path.sep, options.browser)).getDriver(options);

        driver.getCapabilities().then(function(cap) {
          self.browserVersion = cap.get('version');
          self.os = cap.get('platform');
          self.browserName = cap.get('browserName');
        });

        // fetch the URL and wait until the load event ends or we get the time out
        log.log('info', 'Fetch ' + options.url);
        driver.get(options.url);
        driver.wait(function() {
          return driver.executeScript('return window.performance.timing.loadEventEnd>0').then(function(
            fetched) {
            return fetched;
          });
        }, 60000);

        // lets run all scripts
        var promises = [];
        self.scripts.forEach(function(script) {
          promises.push(driver.executeScript(script));
        });<commit_msg>Simplify code to find browser driver.<commit_after>/**
 * Browsertime (http://www.browsertime.com)
 * Copyright (c) 2014, <NAME>, <NAME>
 * and other contributors
 * Released under the Apache 2.0 License
 */
var stats = require('./statistics'),
  fs = require('fs'),
  async = require('async'),
  path = require('path'),
  MobProxy = require('./mobproxy'),
  log = require('winston'),
  url = require('url');

function Browsertime() {
  this.result = [];
  this.scripts = [];
  this.browserVersion = '';
  this.os = '';
  this.browserName = '';
  this.userAgent = '';
}

Browsertime.prototype.fetch = function(options, cb) {

  cb = cb || function() {};

  this._setupDefaults(options);
  this._populateScripts(options);
  this._setupLogging(options);

  if (options.useProxy === true) {
    // setup the mob proxy
    var mobProxy = new MobProxy({
      port: options.mobProxyPort, // make configurable
      proxyPort: options.proxyPort,
      domain: url.parse(options.url).hostname,
      headers: options.headers,
      basicAuth: options.basicAuth,
      limit: options.connectionRaw,
      proxySleepBeforeStart: options.proxySleepBeforeStart
    });

    log.log('info', 'Start the proxy [' + options.mobProxyPort + '], will wait ' + options.proxySleepBeforeStart +
      ' ms');
    var self = this;
    mobProxy.start(function() {
      self._do(mobProxy, options, cb);
    });
  }
  // if we don't use the proxy
  else {
    this._do(undefined, options, cb);
  }
};

Browsertime.prototype._do = function(mobProxy, options, cb) {
  var browserRuns = [];
  var self = this;
  for (var i = 0; i < options.runs; i++) {
    browserRuns.push(self._collectFromTheBrowser(mobProxy, options));
  }

  async.series(browserRuns,
    function(err, results) {
      var data = self._getFormattedResult(options);
      self._saveToFile(data, options, mobProxy, cb);
    });
};

Browsertime.prototype._collectFromTheBrowser = function(mobProxy, options) {
  var self = this;

  return function(callback) {
    async.series([

        function(callback) {
          if (options.useProxy === true) {
            mobProxy.newPage('myname', callback);
          } else {
            callback();
          }
        },
        function(callback) {
          if (options.useProxy === true) {
            mobProxy.clearDNS(callback);
          } else {
            callback();
          }
        }
      ],
      function(err, results) {
        var driver = require(path.join(__dirname, 'browsers', options.browser)).getDriver(options);

        driver.getCapabilities().then(function(cap) {
          self.browserVersion = cap.get('version');
          self.os = cap.get('platform');
          self.browserName = cap.get('browserName');
        });

        // fetch the URL and wait until the load event ends or we get the time out
        log.log('info', 'Fetch ' + options.url);
        driver.get(options.url);
        driver.wait(function() {
          return driver.executeScript('return window.performance.timing.loadEventEnd>0').then(function(
            fetched) {
            return fetched;
          });
        }, 60000);

        // lets run all scripts
        var promises = [];
        self.scripts.forEach(function(script) {
          promises.push(driver.executeScript(script));
        });
{'fraction_non_alphanumeric': 0.14279210925644917, 'fraction_numerical': 0.0039453717754172985, 'mean_word_length': 2.5801195002715915, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4074815', 'n_tokens_mistral': 1974, 'n_tokens_neox': 1925, 'n_words': 414}