<commit_before>import fetch from 'node-fetch'
import parsers from 'www-authenticate/lib/parsers'

function basicAuth(username, password) {
  return `Basic ${new Buffer(`${username}:${password}`).toString('base64')}`
}

function tryJSON(input, url = '') {
  try {
    return JSON.parse(input)
  } catch (ex) {
    throw new Error('Cannot parse JSON from ' + input + ' for ' + url)
  }
}

async function getToken(challenge, creds) {
  const res = await fetch(`${challenge.parms.realm}?service=${challenge.parms.service}&scope=${challenge.parms.scope}`, {
    headers: {
      'Authorization': basicAuth(creds.username, creds.password),
    },
  })

  if (res.status === 401) {
    throw new Error('Invalid authentication')
  }

  return (await res.json()).token
}

async function registryRequest(endpoint, opts) {
  const creds = opts.credentials
  const url = `${opts.registry}/v2/${endpoint}`

  const initialAuth = creds != null
    ? basicAuth(creds.username, creds.password)
    : undefined

  let res = await fetch(url, {
    headers: {
      'Authorization': initialAuth,
      'Content-Type': 'application/vnd.docker.distribution.manifest.list.v2+json',
    },
  })

  if (res.status === 401) {
    const authChallenge = new parsers.WWW_Authenticate(res.headers.get('WWW-Authenticate'))

    if (authChallenge.scheme === 'Basic') {
      throw new Error('Invalid authentication')
    } else if (authChallenge.scheme === 'Bearer') {
      if (!creds) {
        throw new Error('No credentials provided but are required by the registry')
      }

      const token = await getToken(authChallenge, creds)

      res = await fetch(url, {
        headers: {
          'Authorization': `Bearer ${token}`,
          'Content-Type': 'application/vnd.docker.distribution.manifest.list.v2+json',
        },
      })<commit_msg>Support bearer auth without credentials (required for docker hub)<commit_after>import fetch from 'node-fetch'
import parsers from 'www-authenticate/lib/parsers'

function basicAuth(username, password) {
  return `Basic ${new Buffer(`${username}:${password}`).toString('base64')}`
}

function tryJSON(input, url = '') {
  try {
    return JSON.parse(input)
  } catch (ex) {
    throw new Error('Cannot parse JSON from ' + input + ' for ' + url)
  }
}

async function getToken(challenge, creds) {
  const auth = creds != null
    ? basicAuth(creds.username, creds.password)
    : undefined

  const res = await fetch(`${challenge.parms.realm}?service=${challenge.parms.service}&scope=${challenge.parms.scope}`, {
    headers: {
      'Authorization': auth,
    },
  })

  if (res.status === 401) {
    throw new Error('Invalid authentication')
  }

  return (await res.json()).token
}

async function registryRequest(endpoint, opts) {
  const creds = opts.credentials
  const url = `${opts.registry}/v2/${endpoint}`

  const initialAuth = creds != null
    ? basicAuth(creds.username, creds.password)
    : undefined

  let res = await fetch(url, {
    headers: {
      'Authorization': initialAuth,
      'Content-Type': 'application/vnd.docker.distribution.manifest.list.v2+json',
    },
  })

  if (res.status === 401) {
    const authChallenge = new parsers.WWW_Authenticate(res.headers.get('WWW-Authenticate'))

    if (authChallenge.scheme === 'Basic') {
      throw new Error('Invalid authentication')
    } else if (authChallenge.scheme === 'Bearer') {
      const token = await getToken(authChallenge, creds)

      res = await fetch(url, {
        headers: {
          'Authorization': `Bearer ${token}`,
          'Content-Type': 'application/vnd.docker.distribution.manifest.list.v2+json',
        },
      })
{'fraction_non_alphanumeric': 0.1471718835804503, 'fraction_numerical': 0.0060406370126304225, 'mean_word_length': 3.4754299754299756, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2704520', 'n_tokens_mistral': 1174, 'n_tokens_neox': 1107, 'n_words': 251}