<commit_before>/*
 * Proxies all fetch requests to work around
 * CORS issues
 */
import fetch from 'isomorphic-fetch'
let realFetch = fetch

let proxy = "http://localhost:1337/"


let proxyFetch = function(url, options) {
  const tokens = url.split("//")
  const hostAndPath = tokens[1]
  const scheme = tokens[0]
  const host = hostAndPath.split('/')[0]

  if(scheme == "http") {
    throw new Error("Only supports https requests")
  }

  if(host.endsWith("s3.amazonaws.com") ||
     host.endsWith("facebook.com") ||
     host.endsWith("twitter.com") ||
     host.endsWith("github.com")) {
    return realFetch.call(this, proxy + hostAndPath, options)
  } else {
    return realFetch.call(this, url, options)
  }
}


window.fetch = proxyFetch
window.realFetch = realFetch
window.proxyFetch = proxyFetch<commit_msg>include all of aws in proxy fetch<commit_after>/*
 * Proxies all fetch requests to work around
 * CORS issues
 */
import fetch from 'isomorphic-fetch'
let realFetch = fetch

let proxy = "http://localhost:1337/"


let proxyFetch = function(url, options) {
  const tokens = url.split("//")
  const hostAndPath = tokens[1]
  const scheme = tokens[0]
  const host = hostAndPath.split('/')[0]

  if(scheme == "http") {
    throw new Error("Only supports https requests")
  }

  if(host.endsWith("amazonaws.com") ||
     host.endsWith("facebook.com") ||
     host.endsWith("twitter.com") ||
     host.endsWith("github.com")) {
    return realFetch.call(this, proxy + hostAndPath, options)
  } else {
    return realFetch.call(this, url, options)
  }
}


window.fetch = proxyFetch
window.realFetch = realFetch
window.proxyFetch = proxyFetch
{'fraction_non_alphanumeric': 0.1400848999393572, 'fraction_numerical': 0.009096422073984234, 'mean_word_length': 3.796511627906977, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2821917', 'n_tokens_mistral': 532, 'n_tokens_neox': 522, 'n_words': 133}