<commit_before>#!/usr/bin/env node
const fetch = require('node-fetch');
const { groupBy } = require('lodash');
const { setCheckRun } = require('../scripts/check-run');

const { SAUCE_USERNAME, SAUCE_ACCESS_KEY, TRAVIS_JOB_NUMBER } = process.env;
if (!SAUCE_ACCESS_KEY && !SAUCE_USERNAME && !TRAVIS_JOB_NUMBER) {
  console.log(
    `Missing Env Vars, need: SAUCE_USERNAME, SAUCE_ACCESS_KEY, TRAVIS_JOB_NUMBER`,
  );
  process.exit(1);
}

const auth = Buffer.from(`${SAUCE_USERNAME}:${SAUCE_ACCESS_KEY}`).toString(
  'base64',
);

/**
 * @param {string} url
 * @return {Promise<boolean>}
 */
async function isRemoteAssetOk(url) {
  try {
    const { ok, status, headers } = await fetch(url);
    const { 'Content-Length': contentLength } = headers;
    console.log(`isRemoteAssetOk ${url}`, { ok, status, contentLength });
    return ok;
  } catch (err) {
    console.error(`Error on isRemoteAssetOk for ${url}`, err);
    return false;
  }
}

async function collectSauceLabResults(build) {
  try {
    const buildJobs = await fetch(
      // https://wiki.saucelabs.com/display/DOCS/Job+Methods
      `https://saucelabs.com/rest/v1/${SAUCE_USERNAME}/jobs?full=true`,
      {
        headers: {
          Authorization: `Basic ${auth}`,
          Accept: 'application/json',
          'Content-Type': 'application/json',
        },
      },
    )
      .then(res => {
        if (res.ok) {
          console.log(`Set SauceLabs details ok`);
          return res.json();
        } else {
          console.log(`Set SauceLabs details not ok`);
          throw new Error(
            `Could not get SauceLabs job listing ${res.statusText}`,
          );
        }
      })
      .then(jobs => jobs.filter(r => r.build === build));

    const tests = await Promise.all(
      buildJobs.map(async buildJob => {
        // just the file names, not absolute paths
        const assetBaseUrl = `https://assets.saucelabs.com/jobs/${buildJob.id}`;
        /** @type {{ 'sauce-log': string, 'video': string, 'selenium-log': string, screenshots: string[], 'video.mp4': string  }} */
        const assets = await fetch(
          // https://wiki.saucelabs.com/display/DOCS/Job+Methods
          `https://saucelabs.com/rest/v1/${SAUCE_USERNAME}/jobs/${
            buildJob.id
          }/assets`,
          {
            headers: {
              Authorization: `Basic ${auth}`,
              Accept: 'application/json',
            },
          },
        )
          .then(async res => {
            const { ok, status, statusText } = res;
            if (ok) {
              return res.json();
            } else {
              const body = res.text();
              console.log(
                `SauceLabs asset name fetch not ok for ${buildJob.id}: ${body}`,
              );
              throw new Error(
                `Could not get SauceLabs job assets ${status} ${statusText}. ${body}`,
              );
            }
          })
          .then(assetNames => {
            const theAssets = {};
            Object.keys(assetNames).forEach(key => {
              const value = assetNames[key];
              if (key === 'screenshots') {
                theAssets[key] = value.map(name => {
                  return {
                    name,
                    url: `${assetBaseUrl}/${name}`,
                  };
                });
              } else {
                theAssets[key] = `${assetBaseUrl}/${value}`;
              }
            });
            return theAssets;
          })
          .then(async allAssets => {
            // not filtering them yet, just checking; perhaps requesting them will make them work for GitHub
            const validScreenshots = await Promise.all(
              allAssets.screenshots.map(async s => ({
                ...s,
                ok: await isRemoteAssetOk(s.url),
              })),
            );<commit_msg>fix: authing SauceLabs assets for GitHub Checks download<commit_after>#!/usr/bin/env node
const fetch = require('node-fetch');
const { groupBy } = require('lodash');
const { setCheckRun } = require('../scripts/check-run');

const { SAUCE_USERNAME, SAUCE_ACCESS_KEY, TRAVIS_JOB_NUMBER } = process.env;
if (!SAUCE_ACCESS_KEY && !SAUCE_USERNAME && !TRAVIS_JOB_NUMBER) {
  console.log(
    `Missing Env Vars, need: SAUCE_USERNAME, SAUCE_ACCESS_KEY, TRAVIS_JOB_NUMBER`,
  );
  process.exit(1);
}

const auth = Buffer.from(`${SAUCE_USERNAME}:${SAUCE_ACCESS_KEY}`).toString(
  'base64',
);

/**
 * @param {string} url
 * @return {Promise<boolean>}
 */
async function isRemoteAssetOk(url) {
  try {
    const { ok, status, headers } = await fetch(url);
    const { 'Content-Length': contentLength } = headers;
    console.log(`isRemoteAssetOk ${url}`, { ok, status, contentLength });
    return ok;
  } catch (err) {
    console.error(`Error on isRemoteAssetOk for ${url}`, err);
    return false;
  }
}

async function collectSauceLabResults(build) {
  try {
    const buildJobs = await fetch(
      // https://wiki.saucelabs.com/display/DOCS/Job+Methods
      `https://saucelabs.com/rest/v1/${SAUCE_USERNAME}/jobs?full=true`,
      {
        headers: {
          Authorization: `Basic ${auth}`,
          Accept: 'application/json',
          'Content-Type': 'application/json',
        },
      },
    )
      .then(res => {
        if (res.ok) {
          console.log(`Set SauceLabs details ok`);
          return res.json();
        } else {
          console.log(`Set SauceLabs details not ok`);
          throw new Error(
            `Could not get SauceLabs job listing ${res.statusText}`,
          );
        }
      })
      .then(jobs => jobs.filter(r => r.build === build));

    const tests = await Promise.all(
      buildJobs.map(async buildJob => {
        // just the file names, not absolute paths
        const assetBaseUrl = `https://`${SAUCE_USERNAME}:${SAUCE_ACCESS_KEY}`@assets.saucelabs.com/jobs/${buildJob.id}`;
        /** @type {{ 'sauce-log': string, 'video': string, 'selenium-log': string, screenshots: string[], 'video.mp4': string  }} */
        const assets = await fetch(
          // https://wiki.saucelabs.com/display/DOCS/Job+Methods
          `https://saucelabs.com/rest/v1/${SAUCE_USERNAME}/jobs/${
            buildJob.id
          }/assets`,
          {
            headers: {
              Authorization: `Basic ${auth}`,
              Accept: 'application/json',
            },
          },
        )
          .then(async res => {
            const { ok, status, statusText } = res;
            if (ok) {
              return res.json();
            } else {
              const body = res.text();
              console.log(
                `SauceLabs asset name fetch not ok for ${buildJob.id}: ${body}`,
              );
              throw new Error(
                `Could not get SauceLabs job assets ${status} ${statusText}. ${body}`,
              );
            }
          })
          .then(assetNames => {
            const theAssets = {};
            Object.keys(assetNames).forEach(key => {
              const value = assetNames[key];
              if (key === 'screenshots') {
                theAssets[key] = value.map(name => {
                  return {
                    name,
                    url: `${assetBaseUrl}/${name}`,
                  };
                });
              } else {
                theAssets[key] = `${assetBaseUrl}/${value}`;
              }
            });
            return theAssets;
          })
          .then(async allAssets => {
            return allAssets;
            // not filtering them yet, just checking; perhaps requesting them will make them work for GitHub
            const validScreenshots = await Promise.all(
              allAssets.screenshots.map(async s => ({
                ...s,
                ok: await isRemoteAssetOk(s.url),
              })),
            );
{'fraction_non_alphanumeric': 0.14254749458115518, 'fraction_numerical': 0.0015300267754685707, 'mean_word_length': 1.7874911158493247, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 25, 'https://': 10, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5378732', 'n_tokens_mistral': 2365, 'n_tokens_neox': 2315, 'n_words': 493}