<commit_before>
require 'rest-core/middleware'
require 'rest-core/util/json'

class RestCore::JsonResponse
  def self.members; [:json_response]; end
  include RestCore::Middleware

  class ParseError < Json.const_get(:ParseError)
    attr_reader :cause, :body
    def initialize cause, body
      msg = cause.message.force_encoding('utf-8')
      super("#{msg}\nOriginal text: #{body}")
      @cause, @body = cause, body
    end
  end

  JSON_RESPONSE_HEADER = {'Accept' => 'application/json'}.freeze

  def call env, &k
    return app.call(env, &k) if env[DRY]
    return app.call(env, &k) unless json_response(env)

    app.call(env.merge(REQUEST_HEADERS =>
      JSON_RESPONSE_HEADER.merge(env[REQUEST_HEADERS]||{}))){ |response|
        yield(process(response))
      }
  end

  def process response
    body = response[RESPONSE_BODY]
    response.merge(RESPONSE_BODY => Json.decode("[#{body}]").first)
    # [this].first is not needed for yajl-ruby
  rescue Json.const_get(:ParseError) => error
    fail(response, ParseError.new(error, body))
  end
end
<commit_msg>lstrip utf-8 BOM for stackexchange (could they fix this?)<commit_after>
require 'rest-core/middleware'
require 'rest-core/util/json'

class RestCore::JsonResponse
  def self.members; [:json_response]; end
  include RestCore::Middleware

  class ParseError < Json.const_get(:ParseError)
    attr_reader :cause, :body
    def initialize cause, body
      msg = cause.message.force_encoding('utf-8')
      super("#{msg}\nOriginal text: #{body}")
      @cause, @body = cause, body
    end
  end

  JSON_RESPONSE_HEADER = {'Accept' => 'application/json'}.freeze

  def call env, &k
    return app.call(env, &k) if env[DRY]
    return app.call(env, &k) unless json_response(env)

    app.call(env.merge(REQUEST_HEADERS =>
      JSON_RESPONSE_HEADER.merge(env[REQUEST_HEADERS]||{}))){ |response|
        yield(process(response))
      }
  end

  def process response
    # StackExchange returns the problematic BOM! in UTF-8, so we need to
    # strip it or it would break JSON parsers (i.e. yajl-ruby and json)
    body = response[RESPONSE_BODY].to_s.lstrip
    response.merge(RESPONSE_BODY => Json.decode("[#{body}]").first)
    # [this].first is not needed for yajl-ruby
  rescue Json.const_get(:ParseError) => error
    fail(response, ParseError.new(error, body))
  end
end

{'fraction_non_alphanumeric': 0.13071337035454933, 'fraction_numerical': 0.0017086715079026058, 'mean_word_length': 3.8488612836438922, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3820230', 'n_tokens_mistral': 805, 'n_tokens_neox': 789, 'n_words': 185}