<commit_before>'use strict';

const assert = require('../utils/assert');
const setObject = require('../utils/set-object');
const Status = require('../common/status');
const {Metrics} = require('../common/metrics');
const Gatherer = require('./gatherer');

class WebPageTestGatherer extends Gatherer {
  constructor(config, apiHelper, options) {
    super();
    assert(config, 'Parameter config is missing.');
    assert(apiHelper, 'Parameter apiHelper is missing.');

    this.runApiEndpoint = 'https://webpagetest.org/runtest.php';
    this.resultApiEndpoint = 'https://webpagetest.org/jsonResult.php';
    this.apiKey = config.apiKey;
    this.apiHelper = apiHelper;

    options = options || {};
    this.debug = options.debug;

    // TODO: Metadata keys should be standardized.
    this.metadataMap = {
      'testId': 'data.testId',
      'ownerKey': 'data.ownerKey',
      'jsonUrl': 'data.jsonUrl',
      'xmlUrl': 'data.xmlUrl',
      'userUrl': 'data.userUrl',
      'summaryCSV': 'data.summaryCSV',
      'detailCSV': 'data.detailCSV',
    };

    this.metricsMap = {
      // Lighthouse Scores
      'lighthouse.Performance': 'data.median.firstView["lighthouse.Performance"]',
      'lighthouse.ProgressiveWebApp': 'data.median.firstView["lighthouse.ProgressiveWebApp"]',

      // Lighthouse timing metrics
      'lighthouse.FirstContentfulPaint': 'data.median.firstView["lighthouse.Performance.first-contentful-paint"]',
      'lighthouse.FirstMeaningfulPaint': 'data.median.firstView["lighthouse.Performance.first-meaningful-paint"]',
      'lighthouse.SpeedIndex': 'data.median.firstView["lighthouse.Performance.speed-index"]',
      'lighthouse.TimeToInteractive': 'data.median.firstView["lighthouse.Performance.interactive"]',
      'lighthouse.FirstInputDelay': 'data.median.firstView["lighthouse.Performance.max-potential-fid"]',
      'lighthouse.FirstCPUIdle': 'data.median.firstView["lighthouse.Performance.first-cpu-idle"]',
      'lighthouse.TotalBlockingTime': 'data.lighthouse.audits["total-blocking-time"].numericValue',
      'lighthouse.LargestContentfulPaint': 'data.lighthouse.audits.metrics.details.items[0].largestContentfulPaint',
      'lighthouse.CumulativeLayoutShift' : 'data.median.firstView["chromeUserTiming.CumulativeLayoutShift"]',

      // Lighthouse resource size metrics
      'lighthouse.ThirdParty': 'data.lighthouse.audits["third-party-summary"].details.summary.wastedBytes',
      'lighthouse.RenderBlockingResources': 'data.lighthouse.audits["render-blocking-resources"].numericValue',
      'lighthouse.UnusedCSS': 'data.lighthouse.audits["unused-css-rules"].numericValue',
      'lighthouse.UseWebPImages': 'data.lighthouse.audits["uses-webp-images"].numericValue',
      'lighthouse.UseOptimizedImages': 'data.lighthouse.audits["uses-optimized-images"].numericValue',
      'lighthouse.OffscreenImages': 'data.lighthouse.audits["offscreen-images"].numericValue',
      'lighthouse.InstallableManifest': 'data.lighthouse.audits["installable-manifest"].numericValue',
      'lighthouse.ServiceWorker': 'data.lighthouse.audits["service-worker"].numericValue',
      'lighthouse.WorksOffline': 'data.lighthouse.audits["works-offline"].numericValue',

      // WebPageTest Timing metrics
      'TimeToFirstByte': 'data.median.firstView.TTFB',
      'FirstPaint': 'data.median.firstView.render',
      'FirstContentfulPaint': 'data.median.firstView.firstContentfulPaint',
      'FirstMeaningfulPaint': 'data.median.firstView.firstMeaningfulPaint',
      'VisualComplete': 'data.median.firstView.visualComplete',
      'SpeedIndex': 'data.median.firstView.SpeedIndex',
      'LoadEvent': 'data.median.firstView.loadTime',
      'TimeToInteractive': 'data.median.firstView.TTIMeasurementEnd',
      'DOMContentLoaded': 'data.median.firstView.domContentLoadedEventStart',

      // WebPageTest Resource Count metrics
      'Requests': 'data.median.firstView.requestsDoc',
      'DOMElements': 'data.median.firstView.domElements',
      'Connections': 'data.median.firstView.connections',

      // WebPageTest Resource Size metrics
      'TotalSize': 'data.median.firstView.bytesIn',
      'CSS': 'data.median.firstView.breakdown.css.bytes',
      'Fonts': 'data.median.firstView.breakdown.font.bytes',
      'Javascript': 'data.median.firstView.breakdown.js.bytes',
      'Images': 'data.median.firstView.breakdown.image.bytes',
      'Videos': 'data.median.firstView.breakdown.video.bytes'
    };

    let bytesToKb = (x) => Math.round(x / 1000);
    this.metricsConversion = {
      'Size': bytesToKb,
      'CSS': bytesToKb,
      'Fonts': bytesToKb,
      'Javascript': bytesToKb,
      'Images': bytesToKb,
      'Videos': bytesToKb,
      'ThirdParty': bytesToKb,
      'UnusedCSS': bytesToKb,
    };
  }

  run(test, options) {
    assert(test, 'Parameter test is missing.');
    options = options || {};<commit_msg>added TotalSize and CLS Metric support<commit_after>'use strict';

const assert = require('../utils/assert');
const setObject = require('../utils/set-object');
const Status = require('../common/status');
const {Metrics} = require('../common/metrics');
const Gatherer = require('./gatherer');

class WebPageTestGatherer extends Gatherer {
  constructor(config, apiHelper, options) {
    super();
    assert(config, 'Parameter config is missing.');
    assert(apiHelper, 'Parameter apiHelper is missing.');

    this.runApiEndpoint = 'https://webpagetest.org/runtest.php';
    this.resultApiEndpoint = 'https://webpagetest.org/jsonResult.php';
    this.apiKey = config.apiKey;
    this.apiHelper = apiHelper;

    options = options || {};
    this.debug = options.debug;

    // TODO: Metadata keys should be standardized.
    this.metadataMap = {
      'testId': 'data.testId',
      'ownerKey': 'data.ownerKey',
      'jsonUrl': 'data.jsonUrl',
      'xmlUrl': 'data.xmlUrl',
      'userUrl': 'data.userUrl',
      'summaryCSV': 'data.summaryCSV',
      'detailCSV': 'data.detailCSV',
    };

    this.metricsMap = {
      // Lighthouse Scores
      'lighthouse.Performance': 'data.median.firstView["lighthouse.Performance"]',
      'lighthouse.ProgressiveWebApp': 'data.median.firstView["lighthouse.ProgressiveWebApp"]',

      // Lighthouse timing metrics
      'lighthouse.FirstContentfulPaint': 'data.median.firstView["lighthouse.Performance.first-contentful-paint"]',
      'lighthouse.FirstMeaningfulPaint': 'data.median.firstView["lighthouse.Performance.first-meaningful-paint"]',
      'lighthouse.SpeedIndex': 'data.median.firstView["lighthouse.Performance.speed-index"]',
      'lighthouse.TimeToInteractive': 'data.median.firstView["lighthouse.Performance.interactive"]',
      'lighthouse.FirstInputDelay': 'data.median.firstView["lighthouse.Performance.max-potential-fid"]',
      'lighthouse.FirstCPUIdle': 'data.median.firstView["lighthouse.Performance.first-cpu-idle"]',
      'lighthouse.TotalBlockingTime': 'data.lighthouse.audits["total-blocking-time"].numericValue',
      'lighthouse.LargestContentfulPaint': 'data.lighthouse.audits.metrics.details.items[0].largestContentfulPaint',
      'lighthouse.CumulativeLayoutShift' : 'data.median.firstView["chromeUserTiming.CumulativeLayoutShift"]',

      // Lighthouse resource size metrics
      'lighthouse.ThirdParty': 'data.lighthouse.audits["third-party-summary"].details.summary.wastedBytes',
      'lighthouse.RenderBlockingResources': 'data.lighthouse.audits["render-blocking-resources"].numericValue',
      'lighthouse.UnusedCSS': 'data.lighthouse.audits["unused-css-rules"].numericValue',
      'lighthouse.UseWebPImages': 'data.lighthouse.audits["uses-webp-images"].numericValue',
      'lighthouse.UseOptimizedImages': 'data.lighthouse.audits["uses-optimized-images"].numericValue',
      'lighthouse.OffscreenImages': 'data.lighthouse.audits["offscreen-images"].numericValue',
      'lighthouse.InstallableManifest': 'data.lighthouse.audits["installable-manifest"].numericValue',
      'lighthouse.ServiceWorker': 'data.lighthouse.audits["service-worker"].numericValue',
      'lighthouse.WorksOffline': 'data.lighthouse.audits["works-offline"].numericValue',

      // WebPageTest Timing metrics
      'TimeToFirstByte': 'data.median.firstView.TTFB',
      'FirstPaint': 'data.median.firstView.render',
      'FirstContentfulPaint': 'data.median.firstView.firstContentfulPaint',
      'FirstMeaningfulPaint': 'data.median.firstView.firstMeaningfulPaint',
      'VisualComplete': 'data.median.firstView.visualComplete',
      'SpeedIndex': 'data.median.firstView.SpeedIndex',
      'LoadEvent': 'data.median.firstView.loadTime',
      'TimeToInteractive': 'data.median.firstView.TTIMeasurementEnd',
      'DOMContentLoaded': 'data.median.firstView.domContentLoadedEventStart',

      // WebPageTest Resource Count metrics
      'Requests': 'data.median.firstView.requestsDoc',
      'DOMElements': 'data.median.firstView.domElements',
      'Connections': 'data.median.firstView.connections',

      // WebPageTest Resource Size metrics
      'TotalSize': 'data.median.firstView.bytesIn',
      'CSS': 'data.median.firstView.breakdown.css.bytes',
      'Fonts': 'data.median.firstView.breakdown.font.bytes',
      'Javascript': 'data.median.firstView.breakdown.js.bytes',
      'Images': 'data.median.firstView.breakdown.image.bytes',
      'Videos': 'data.median.firstView.breakdown.video.bytes'
    };

    let bytesToKb = (x) => Math.round(x / 1000);
    this.metricsConversion = {
      'TotalSize': bytesToKb,
      'CSS': bytesToKb,
      'Fonts': bytesToKb,
      'Javascript': bytesToKb,
      'Images': bytesToKb,
      'Videos': bytesToKb,
      'ThirdParty': bytesToKb,
      'UnusedCSS': bytesToKb,
    };
  }

  run(test, options) {
    assert(test, 'Parameter test is missing.');
    options = options || {};
{'fraction_non_alphanumeric': 0.15224063842848373, 'fraction_numerical': 0.0010231225700838961, 'mean_word_length': 4.238478027867095, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1527268', 'n_tokens_mistral': 3162, 'n_tokens_neox': 2967, 'n_words': 397}