<commit_before>require 'httparty'
require 'json'

module AWSCosts::Cache
  extend self

  BASE_URI = "http://aws.amazon.com"
  BASE_JSONP_URI = "http://a0.awsstatic.com"

  def get uri, base_uri = BASE_URI, &block
    cache[uri] ||= begin
      yield JSON.parse(HTTParty.get("#{base_uri}#{uri}").body)
     end
  end

  def get_jsonp uri, base_uri = BASE_JSONP_URI, &block
    cache[uri] ||= begin
      yield extract_json_from_callback(HTTParty.get("#{base_uri}#{uri}").body)
    end
  end

  private
  def cache
    @cache ||= {}
  end

  def extract_json_from_callback body
    body.match /^.*callback\((\{.*\})\);$/
    body = $1

    # Handle "json" with keys that are not quoted
    # When we get {foo: "1"} instead of {"foo": "1"}
    # http://stackoverflow.com/questions/2060356/parsing-json-without-quoted-keys
    JSON.parse(body.gsub(/(\w+)\s*:/, '"\1":'))
  end
end<commit_msg>Handle empty body with some retries<commit_after>require 'httparty'
require 'json'

module AWSCosts::Cache
  extend self

  BASE_URI = "http://aws.amazon.com"
  BASE_JSONP_URI = "http://a0.awsstatic.com"

  def get uri, base_uri = BASE_URI, &block
    cache[uri] ||= begin
      yield JSON.parse(HTTParty.get("#{base_uri}#{uri}").body)
     end
  end

  def get_jsonp uri, base_uri = BASE_JSONP_URI, &block
    attempts = 0
    cache[uri] ||= begin
      yield extract_json_from_callback(HTTParty.get("#{base_uri}#{uri}").body)
    rescue NoMethodError
      attempts += 1
      retry if attempts < 5
      raise "Failed to retrieve or parse data for #{base_uri}#{uri}"
    end
  end

  private
  def cache
    @cache ||= {}
  end

  def extract_json_from_callback body
    body.match /^.*callback\((\{.*\})\);$/
    body = $1

    # Handle "json" with keys that are not quoted
    # When we get {foo: "1"} instead of {"foo": "1"}
    # http://stackoverflow.com/questions/2060356/parsing-json-without-quoted-keys
    JSON.parse(body.gsub(/(\w+)\s*:/, '"\1":'))
  end
end
{'fraction_non_alphanumeric': 0.15926493108728942, 'fraction_numerical': 0.013782542113323124, 'mean_word_length': 3.1702127659574466, 'pattern_counts': {'":': 4, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1510530', 'n_tokens_mistral': 738, 'n_tokens_neox': 711, 'n_words': 163}