<commit_before># https://gist.github.com/letronje/437b4a72225eb53366c6
#
module PaginatedHer::Middleware
  class Parser < Faraday::Response::Middleware

    def on_complete(env)
      response = env[:response]
      if response.success?
        json = parsed_response(env)
        errors = json.delete(:errors) || {}
        metadata = json.delete(:metadata) || []
        body = {:data => json, :errors => errors, :metadata => metadata}
        if env[:response_headers]["X-Pagination"]
          body[:pagination] = JSON.parse(env[:response_headers]["X-Pagination"], :symbolize_names => true)
        end
        env[:body] = body
      else
        if response.status == 401
          raise PaginatedHer::AuthRequired
        else
          raise PaginatedHer::RequestFailed
        end
      end
    end
    
    def parsed_response(env)
      begin
        JSON.parse(env[:body], :symbolize_names => true)
      rescue => e
        Rails.logger.warn "JSON parse failure: #{e}"
        {}
      end
    end
    
  end
end<commit_msg>handle 404s separately from errors<commit_after># https://gist.github.com/letronje/437b4a72225eb53366c6
#
module PaginatedHer::Middleware
  class Parser < Faraday::Response::Middleware

    def on_complete(env)
      response = env[:response]
      if response.success?
        json = parsed_response(env)
        errors = json.delete(:errors) || {}
        metadata = json.delete(:metadata) || []
        body = {:data => json, :errors => errors, :metadata => metadata}
        if env[:response_headers]["X-Pagination"]
          body[:pagination] = JSON.parse(env[:response_headers]["X-Pagination"], :symbolize_names => true)
        end
        env[:body] = body
      else
        if response.status == 404
          body = {:data => {}, :errors => "Resource Not Found", :metadata => json.delete(:metadata) || []}
          env[:body] = body
        if response.status == 401
          raise PaginatedHer::AuthRequired
        else
          raise PaginatedHer::RequestFailed
        end
      end
    end
    
    def parsed_response(env)
      begin
        JSON.parse(env[:body], :symbolize_names => true)
      rescue => e
        Rails.logger.warn "JSON parse failure: #{e}"
        {}
      end
    end
    
  end
end
{'fraction_non_alphanumeric': 0.12411660777385158, 'fraction_numerical': 0.018551236749116608, 'mean_word_length': 2.375558867362146, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 18, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1995373', 'n_tokens_mistral': 691, 'n_tokens_neox': 657, 'n_words': 144}