<commit_before>'use strict';

const clone = require('lodash.clonedeep');
const get = require('lodash.get');
const WebPageTest = require('webpagetest');
const { promisify } = require('util');

module.exports = {
  async analyzeUrl(url, storageManager, log, wptOptions) {
    const wptClient = new WebPageTest(wptOptions.host, wptOptions.key);
    wptOptions.firstViewOnly = !wptOptions.includeRepeatView;
    let urlOrScript = url;

    log.info('Sending url ' + url + ' to test on ' + wptOptions.host);
    if (wptOptions.script) {
      urlOrScript = wptOptions.script.split('{{{URL}}}').join(url);
    }

    // Setup WebPageTest methods
    const runTest = promisify(wptClient.runTest.bind(wptClient));
    const getHARData = promisify(wptClient.getHARData.bind(wptClient));
    const getScreenshotImage = promisify(
      wptClient.getScreenshotImage.bind(wptClient)
    );
    const getWaterfallImage = promisify(
      wptClient.getWaterfallImage.bind(wptClient)
    );
    const getChromeTraceData = promisify(
      wptClient.getChromeTraceData.bind(wptClient)
    );

    // See https://github.com/sitespeedio/sitespeed.io/issues/1367
    const options = clone(wptOptions);

    try {
      const data = await runTest(urlOrScript, options);
      const id = data.data.id;
      log.info('Got %s analysed with id %s from %s', url, id, options.host);
      log.verbose('Got JSON from WebPageTest :%:2j', data);

      // Something failed with WebPageTest but how should we handle that?
      // Also, this doesn't contain every failure case e.g. successfulFV/RVRuns=0 we should include it
      if (data.statusCode !== 200) {
        log.error(
          'The test got status code %s from WebPageTest with %s. Checkout %s to try to find the original reason.',
          data.statusCode,
          data.statusText,
          get(data, 'data.summary')
        );
      } else {
        log.info('WebPageTest result at: ' + data.data.summary);
      }
<commit_msg>Check for multi step WPT result and log that it doesn't work (#2621)<commit_after>'use strict';

const clone = require('lodash.clonedeep');
const get = require('lodash.get');
const WebPageTest = require('webpagetest');
const { promisify } = require('util');

module.exports = {
  async analyzeUrl(url, storageManager, log, wptOptions) {
    const wptClient = new WebPageTest(wptOptions.host, wptOptions.key);
    wptOptions.firstViewOnly = !wptOptions.includeRepeatView;
    let urlOrScript = url;

    log.info('Sending url ' + url + ' to test on ' + wptOptions.host);
    if (wptOptions.script) {
      urlOrScript = wptOptions.script.split('{{{URL}}}').join(url);
    }

    // Setup WebPageTest methods
    const runTest = promisify(wptClient.runTest.bind(wptClient));
    const getHARData = promisify(wptClient.getHARData.bind(wptClient));
    const getScreenshotImage = promisify(
      wptClient.getScreenshotImage.bind(wptClient)
    );
    const getWaterfallImage = promisify(
      wptClient.getWaterfallImage.bind(wptClient)
    );
    const getChromeTraceData = promisify(
      wptClient.getChromeTraceData.bind(wptClient)
    );

    // See https://github.com/sitespeedio/sitespeed.io/issues/1367
    const options = clone(wptOptions);

    try {
      const data = await runTest(urlOrScript, options);
      const id = data.data.id;
      log.info('Got %s analysed with id %s from %s', url, id, options.host);
      log.verbose('Got JSON from WebPageTest :%:2j', data);

      // Something failed with WebPageTest but how should we handle that?
      // Also, this doesn't contain every failure case e.g. successfulFV/RVRuns=0 we should include it
      if (data.statusCode !== 200) {
        log.error(
          'The test got status code %s from WebPageTest with %s. Checkout %s to try to find the original reason.',
          data.statusCode,
          data.statusText,
          get(data, 'data.summary')
        );
      } else {
        log.info('WebPageTest result at: ' + data.data.summary);
      }

      if (
        data.data.median.firstView.numSteps &&
        data.data.median.firstView.numSteps > 1
      ) {
        // MULTISTEP
        log.info(
          "Sitespeed.io doesn't support multi step WebPageTest scripting at the moment. Either use sitespeed.io scripting (https://www.sitespeed.io/documentation/sitespeed.io/scripting/) or help us implement support for WebPageTest in https://github.com/sitespeedio/sitespeed.io/issues/2620"
        );
        return;
      }

{'fraction_non_alphanumeric': 0.1164506034868127, 'fraction_numerical': 0.006034868126955744, 'mean_word_length': 3.046112115732369, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 4, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1062760', 'n_tokens_mistral': 1457, 'n_tokens_neox': 1406, 'n_words': 339}