<commit_before>/* eslint-disable max-len */
/* eslint-disable no-confusing-arrow */
import { camelizeKeys } from 'humps'
import * as ACTION_TYPES from '../constants/action_types'
import { get } from 'lodash'
import { refreshAuthenticationToken } from '../actions/authentication'

let requesterIsPaused = false
let requestQueue = []
let runningFetches = {}

const defaultHeaders = {
  Accept: 'application/json',
  'Content-Type': 'application/json',
}

function getAuthToken(accessToken) {
  return {
    Authorization: `Bearer ${accessToken}`,
  }
}

function getPostJsonHeader(accessToken) {
  return {
    ...defaultHeaders,
    ...getAuthToken(accessToken),
  }
}

function getGetHeader(accessToken) {
  return getAuthToken(accessToken)
}

function getHeadHeader(accessToken, lastCheck) {
  return {
    ...getAuthToken(accessToken),
    'If-Modified-Since': lastCheck,
  }
}

function updateRunningFetches(response) {
  if (runningFetches[response.url]) {
    delete runningFetches[response.url]
  } else {
    runningFetches = {}
  }
}

function checkStatus(response) {
  updateRunningFetches(response)
  if (response.ok) {
    return response
  }
  const error = new Error(response.statusText)
  error.response = response
  throw error
}

function parseLink(linksHeader) {
  if (!linksHeader) { return {} }
  const result = {}
  const entries = linksHeader.split(',')
  // compile regular expressions ahead of time for efficiency
  const relsRegExp = /\brel="?([^"]+)"?\s*;?/
  const keysRegExp = /(\<KEY>
  const sourceRegExp = /^<(.*)>/
  for (let entry of entries) {
    entry = entry.trim()
    const rels = relsRegExp.exec(entry)
    if (rels) {
      const keys = rels[1].match(keysRegExp)<commit_msg>Grab response url from headers when fetching<commit_after>/* eslint-disable max-len */
/* eslint-disable no-confusing-arrow */
import { camelizeKeys } from 'humps'
import * as ACTION_TYPES from '../constants/action_types'
import { get } from 'lodash'
import { refreshAuthenticationToken } from '../actions/authentication'

let requesterIsPaused = false
let requestQueue = []
let runningFetches = {}

const defaultHeaders = {
  Accept: 'application/json',
  'Content-Type': 'application/json',
}

function getAuthToken(accessToken) {
  return {
    Authorization: `Bearer ${accessToken}`,
  }
}

function getPostJsonHeader(accessToken) {
  return {
    ...defaultHeaders,
    ...getAuthToken(accessToken),
  }
}

function getGetHeader(accessToken) {
  return getAuthToken(accessToken)
}

function getHeadHeader(accessToken, lastCheck) {
  return {
    ...getAuthToken(accessToken),
    'If-Modified-Since': lastCheck,
  }
}

function updateRunningFetches(response) {
  const responseUrl = response.url && response.url.length ? response.url : response.headers.get('X-Request-Url')
  if (runningFetches[responseUrl]) {
    delete runningFetches[responseUrl]
  } else {
    runningFetches = {}
  }
}

function checkStatus(response) {
  updateRunningFetches(response)
  if (response.ok) {
    return response
  }
  const error = new Error(response.statusText)
  error.response = response
  throw error
}

function parseLink(linksHeader) {
  if (!linksHeader) { return {} }
  const result = {}
  const entries = linksHeader.split(',')
  // compile regular expressions ahead of time for efficiency
  const relsRegExp = /\brel="?([^"]+)"?\s*;?/
  const keysRegExp = /(\<KEY>
  const sourceRegExp = /^<(.*)>/
  for (let entry of entries) {
    entry = entry.trim()
    const rels = relsRegExp.exec(entry)
    if (rels) {
      const keys = rels[1].match(keysRegExp)
{'fraction_non_alphanumeric': 0.1330532212885154, 'fraction_numerical': 0.0005602240896358543, 'mean_word_length': 3.9944055944055945, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5080797', 'n_tokens_mistral': 1145, 'n_tokens_neox': 1133, 'n_words': 253}