<commit_before>module Misty
  module HTTP
    module Request
      def decode?(response)
        return false if response.body.nil? || response.body.empty?
        if @ether_content_type
          content_type = @ether_content_type
          @ether_content_type = nil
        else
          content_type = @content_type
        end
        if content_type != :json && response.code =~ /2??/ && !response.is_a?(Net::HTTPNoContent) \
          && !response.is_a?(Net::HTTPResetContent) && response.header['content-type'] \
          && response.header['content-type'].include?('application/json')
          return true
        end
        false
      end

      def http(request)
        request['X-Auth-Token'] = @auth.get_token

        Misty::HTTP::NetHTTP.http_request(
          @uri, ssl_verify_mode: @ssl_verify_mode, log: @log
        ) do |connection|
          response = connection.request(request)
          response.body = JSON.parse(response.body) if decode?(response)<commit_msg>Add list of mime/types JSON decode<commit_after>module Misty
  module HTTP
    module Request

      DECODE_TO_JSON = ["application/json", "application/octet-stream"].freeze

      def decode?(response)
        return false if response.body.nil? || response.body.empty?
        if @ether_content_type
          content_type = @ether_content_type
          @ether_content_type = nil
        else
          content_type = @content_type
        end
        if content_type != :json && response.code =~ /2??/ \
          && !response.is_a?(Net::HTTPNoContent) \
          && !response.is_a?(Net::HTTPResetContent) \
          && response.header['content-type'] && decode_to_json?(response.header['content-type'])
          return true
        end
        false
      end

      def decode_to_json?(content_type)
        DECODE_TO_JSON.each do |type|
          return true if content_type.include?(type)
        end
        false
      end

      def http(request)
        request['X-Auth-Token'] = @auth.get_token

        Misty::HTTP::NetHTTP.http_request(
          @uri, ssl_verify_mode: @ssl_verify_mode, log: @log
        ) do |connection|
          response = connection.request(request)
          response.body = JSON.parse(response.body) if decode?(response)
{'fraction_non_alphanumeric': 0.11180124223602485, 'fraction_numerical': 0.0008873114463176575, 'mean_word_length': 2.325958702064897, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5737877', 'n_tokens_mistral': 649, 'n_tokens_neox': 651, 'n_words': 129}