<commit_before>/**
 * Sitespeed.io - How speedy is your site? (http://www.sitespeed.io)
 * Copyright (c) 2014, <NAME>, <NAME>
 * and other contributors
 * Released under the Apache 2.0 License
 */

var path = require('path'),
  util = require('../util/util'),
  inspect = require('util').inspect,
  fs = require('fs'),
  WebPageTest = require('webpagetest'),
  winston = require('winston'),
  async = require('async');

module.exports = {
  analyze: function(urls, config, callback) {
    var log = winston.loggers.get('sitespeed.io');
    var wptResultDir = path.join(config.run.absResultDir, config.dataDir, 'webpagetest');
    fs.mkdir(wptResultDir, function(err) {

      if (err) {
        log.log('error', 'Could not create the WPT result dir: ' + wptResultDir + ' ' + err);
        callback(err, {
          'type': 'webpagetest',
          'data': {},
          'errors': {}
        });
      } else {
        var queue = async.queue(analyzeUrl, config.threads);
        var errors = {};
        var pageData = {};

        var wpt = ((config.wptKey) ? new WebPageTest(config.wptHost, config.wptKey) :
          new WebPageTest(config.wptHost));

        urls.forEach(function(u) {
          queue.push({
            'url': u,
            'wpt': wpt,
            'config': config
          }, function(err, data) {
            if (err) {
              log.log('error', 'Error running WebPageTest: ' + err);
              errors[u] = err;
            } else {
              //Always return an array for consistency in how we process aggregators
              if (!Array.isArray(data.wpt.response.data.run)) {
                data.wpt.response.data.run = [data.wpt.response.data.run];
              }
              pageData[u] = data;
            }
          });
        });

        queue.drain = function() {
          callback(undefined, {
            'type': 'webpagetest',
            'data': pageData,
            'errors': errors
          });
        };
      }

    });
  }
};

function analyzeUrl(args, asyncDoneCallback) {
  var url = args.url;
  var wpt = args.wpt;
  var config = args.config;
  var log = winston.loggers.get('sitespeed.io');

  var wptOptions = ({
    pollResults: 10,
    timeout: 600,
    firstViewOnly: false,
    runs: config.no,
    private: true,
    aftRenderingTime: true,
    location: 'Dulles:Chrome',
    video: true
  });

  // set basic auth if it is configured
  if (config.basicAuth) {
    wptOptions.login = config.basicAuth.split(':')[0];
    wptOptions.password = <PASSWORD>.split(':')[1];
  }

  // then add the ones from the supplied configuration
  if (config.wptConfig) {
    Object.keys(config.wptConfig).forEach(function(key) {
      wptOptions[key] = config.wptConfig[key];
    });
  }
  log.info('Running WebPageTest %s', url, wptOptions);

  wpt.runTest(url, wptOptions, function(err, data) {

    if (err) {
      log.error('WebPageTest couldn\'t fetch info for url %s (%s)', url, inspect(err));
      return asyncDoneCallback(err);
    }

    var id = data.response.data.testId;
    var har;

    async.parallel({
        fetchWaterfall: function(cb) {
          var waterfallOptions = {<commit_msg>added extra check that WPT returns a response<commit_after>/**
 * Sitespeed.io - How speedy is your site? (http://www.sitespeed.io)
 * Copyright (c) 2014, <NAME>, <NAME>
 * and other contributors
 * Released under the Apache 2.0 License
 */

var path = require('path'),
  util = require('../util/util'),
  inspect = require('util').inspect,
  fs = require('fs'),
  WebPageTest = require('webpagetest'),
  winston = require('winston'),
  async = require('async');

module.exports = {
  analyze: function(urls, config, callback) {
    var log = winston.loggers.get('sitespeed.io');
    var wptResultDir = path.join(config.run.absResultDir, config.dataDir, 'webpagetest');
    fs.mkdir(wptResultDir, function(err) {

      if (err) {
        log.log('error', 'Could not create the WPT result dir: ' + wptResultDir + ' ' + err);
        callback(err, {
          'type': 'webpagetest',
          'data': {},
          'errors': {}
        });
      } else {
        var queue = async.queue(analyzeUrl, config.threads);
        var errors = {};
        var pageData = {};

        var wpt = ((config.wptKey) ? new WebPageTest(config.wptHost, config.wptKey) :
          new WebPageTest(config.wptHost));

        urls.forEach(function(u) {
          queue.push({
            'url': u,
            'wpt': wpt,
            'config': config
          }, function(err, data) {
            if (err) {
              log.log('error', 'Error running WebPageTest: ' + err);
              errors[u] = err;
            } else {
              //Always return an array for consistency in how we process aggregators
              if (!Array.isArray(data.wpt.response.data.run)) {
                data.wpt.response.data.run = [data.wpt.response.data.run];
              }
              pageData[u] = data;
            }
          });
        });

        queue.drain = function() {
          callback(undefined, {
            'type': 'webpagetest',
            'data': pageData,
            'errors': errors
          });
        };
      }

    });
  }
};

function analyzeUrl(args, asyncDoneCallback) {
  var url = args.url;
  var wpt = args.wpt;
  var config = args.config;
  var log = winston.loggers.get('sitespeed.io');

  var wptOptions = ({
    pollResults: 10,
    timeout: 600,
    firstViewOnly: false,
    runs: config.no,
    private: true,
    aftRenderingTime: true,
    location: 'Dulles:Chrome',
    video: true
  });

  // set basic auth if it is configured
  if (config.basicAuth) {
    wptOptions.login = config.basicAuth.split(':')[0];
    wptOptions.password = config.basicAuth.split(':')[1];
  }

  // then add the ones from the supplied configuration
  if (config.wptConfig) {
    Object.keys(config.wptConfig).forEach(function(key) {
      wptOptions[key] = config.wptConfig[key];
    });
  }
  log.info('Running WebPageTest %s', url, wptOptions);

  wpt.runTest(url, wptOptions, function(err, data) {

    if (err) {
      log.error('WebPageTest couldn\'t fetch info for url %s (%s)', url, inspect(err));
      return asyncDoneCallback(err);
    }

    if (!data.response) {
      log.error('WebPageTest didn\'t deliver a response for url %s (%s)', url, data);
      return asyncDoneCallback(new Error('No response from WPT.'));
    }

    var id = data.response.data.testId;
    var har;

    async.parallel({
        fetchWaterfall: function(cb) {
          var waterfallOptions = {
{'fraction_non_alphanumeric': 0.14573477796429116, 'fraction_numerical': 0.0039676484053105445, 'mean_word_length': 2.249380267724343, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5494241', 'n_tokens_mistral': 2104, 'n_tokens_neox': 2058, 'n_words': 474}