<commit_before>'use strict';

const Stats = require('fast-stats').Stats,
  urlParser = require('url'),
  log = require('intel'),
  statsHelpers = require('../../support/statsHelpers'),
  isNotEmpty = require('../../support/util').isNotEmpty,
  reduce = require('lodash.reduce');

const timingNames = ['blocked', 'dns', 'connect', 'ssl', 'send', 'wait', 'receive'];

const domains = {};

function parseDomainName(url) {
  return urlParser.parse(url).hostname;
}

function getDomainEntry(domainName) {
  let domain = domains[domainName];
  if (!domain) {
    domain = {
      domainName,
      requestCount: 0,
      totalTime: new Stats(),
      blocked: new Stats(),
      dns: new Stats(),
      connect: new Stats(),
      ssl: new Stats(),
      send: new Stats(),
      wait: new Stats(),
      receive: new Stats()
    };
    domains[domainName] = domain;
  }

  return domain;
}

module.exports = {
  addToAggregate(har) {
    const firstPageId = har.log.pages[0].id;

    har.log.entries.forEach((entry) => {
      if (entry.pageref !== firstPageId) {
        // Only pick the first request out of multiple runs.
        return;
      }

      const domainName = parseDomainName(entry.request.url),
        domain = getDomainEntry(domainName),
        time = entry.time;

      domain.domainName = domainName;
      domain.requestCount++;
      if (time && time != -1) {
        domain.totalTime.push(entry.time);
      } else {
        log.debug('Missing time from har entry for url: ' + entry.request.url);
      }

      timingNames.forEach((name) => {
        const timing = entry.timings[name];

        if (timing && timing != -1) {
          domain[name].push(timing);
        }
      });
    });
  },
  summarize() {
    return reduce(domains, (summary, domainStats, domainName) => {
      const domainSummary = {
        domainName: domainStats.domainName,
        requestCount: domainStats.requestCount,
        totalTime: statsHelpers.summarizeStats(domainStats.totalTime)
      };
<commit_msg>Account for entry.time missing on har files.<commit_after>'use strict';

const Stats = require('fast-stats').Stats,
  urlParser = require('url'),
  log = require('intel'),
  statsHelpers = require('../../support/statsHelpers'),
  isNotEmpty = require('../../support/util').isNotEmpty,
  reduce = require('lodash.reduce');

const timingNames = ['blocked', 'dns', 'connect', 'ssl', 'send', 'wait', 'receive'];

const domains = {};

function parseDomainName(url) {
  return urlParser.parse(url).hostname;
}

function getDomainEntry(domainName) {
  let domain = domains[domainName];
  if (!domain) {
    domain = {
      domainName,
      requestCount: 0,
      totalTime: new Stats(),
      blocked: new Stats(),
      dns: new Stats(),
      connect: new Stats(),
      ssl: new Stats(),
      send: new Stats(),
      wait: new Stats(),
      receive: new Stats()
    };
    domains[domainName] = domain;
  }

  return domain;
}

function isValidTiming(timing) {
  // The HAR format uses -1 to indicate invalid/missing timings
  return (typeof timing === 'number' && timing !== -1);
}

module.exports = {
  addToAggregate(har) {
    const firstPageId = har.log.pages[0].id;

    har.log.entries.forEach((entry) => {
      if (entry.pageref !== firstPageId) {
        // Only pick the first request out of multiple runs.
        return;
      }

      const domainName = parseDomainName(entry.request.url),
        domain = getDomainEntry(domainName),
        totalTime = entry.time;

      domain.domainName = domainName;
      domain.requestCount++;
      if (isValidTiming(totalTime)) {
        domain.totalTime.push(totalTime);
      } else {
        log.debug('Missing time from har entry for url: ' + entry.request.url);
      }

      timingNames.forEach((name) => {
        const timing = entry.timings[name];

        if (isValidTiming(timing)) {
          domain[name].push(timing);
        }
      });
    });
  },
  summarize() {
    return reduce(domains, (summary, domainStats, domainName) => {
      const domainSummary = {
        domainName: domainStats.domainName,
        requestCount: domainStats.requestCount
      };

      const stats = statsHelpers.summarizeStats(domainStats.totalTime);
      if (isNotEmpty(stats)) {
        domainSummary.totalTime = stats;
      }
{'fraction_non_alphanumeric': 0.1436875145315043, 'fraction_numerical': 0.001860032550569635, 'mean_word_length': 2.724675324675325, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5195786', 'n_tokens_mistral': 1302, 'n_tokens_neox': 1275, 'n_words': 294}