<commit_before>require 'faraday'
require 'faraday_middleware'
require 'net/http'
require 'excon'
require 'uri'

module Networkable
  extend ActiveSupport::Concern

  included do
    def get_result(url, options={})
      options[:content_type] ||= 'json'
      options[:headers] ||= {}
      options[:headers] = set_request_headers(url, options)

      conn = faraday_conn(options[:content_type], options)

      conn.options[:timeout] = options[:timeout] || DEFAULT_TIMEOUT

      if options[:data]
        response = conn.post url, {}, options[:headers] do |request|
          request.body = options[:data]
        end
      else
        response = conn.get url, {}, options[:headers]
      end
      # set number of available API calls for agents
      if options[:agent_id].present?
        agent = Agent.where(id: options[:agent_id]).first
        agent.update_attributes(rate_limit_remaining: get_rate_limit_remaining(response.headers),
                                rate_limit_reset: get_rate_limit_reset(response.headers),
                                last_response: Time.zone.now)
      end
      # parsing by content type is not reliable, so we check the response format
      if is_json?(response.body)
        JSON.parse(response.body)
      elsif is_xml?(response.body)
        Hash.from_xml(response.body)
      else
        response.body
      end
    rescue *NETWORKABLE_EXCEPTIONS => e
      rescue_faraday_error(url, e, options)
    end

    def set_request_headers(url, options)
      options[:headers] ||= {}
      options[:headers]['Host'] = URI.parse(url).host

      if options[:bearer].present?
        options[:headers]['Authorization'] = "Bearer #{options[:bearer]}"
      elsif options[:token].present?
        options[:headers]["Authorization"] = "Token token=#{options[:token]}"
      elsif options[:username].present?
        basic = Base64.encode64("#{options[:username]}:#{options[:password].to_s}")
        options[:headers]["Authorization"] = "Basic #{basic}"
      end

      options[:headers]
    end

    def faraday_conn(content_type = 'json', options = {})
      content_types = { "html" => 'text/html; charset=UTF-8',
                        "xml" => 'application/xml',
                        "json" => 'application/json' }
      accept_header = content_types.fetch(content_type, content_type)
      limit = options[:limit] || 10

      Faraday.new do |c|
        c.headers['Accept'] = accept_header
        c.headers['User-Agent'] = "Lagotto - http://#{ENV['SERVERNAME']}"
        c.use      FaradayMiddleware::FollowRedirects, limit: limit, cookie: :all
        c.request  :multipart
        c.request  :json if accept_header == 'application/json'
        c.use      Faraday::Response::RaiseError
        c.adapter  Faraday.default_adapter
      end
    end

    def rescue_faraday_error(url, error, options={})
      if error.is_a?(Faraday::ResourceNotFound)
        not_found_error(url, error, options)
      else
        details = nil
        headers = {}

        if error.is_a?(Faraday::Error::TimeoutError)
          status = 408
        elsif error.respond_to?('status')<commit_msg>fixed basic authentication<commit_after>require 'faraday'
require 'faraday_middleware'
require 'net/http'
require 'excon'
require 'uri'

module Networkable
  extend ActiveSupport::Concern

  included do
    def get_result(url, options={})
      options[:headers] ||= {}
      options[:headers] = set_request_headers(url, options)

      conn = faraday_conn(options)

      conn.options[:timeout] = options[:timeout] || DEFAULT_TIMEOUT

      if options[:data]
        response = conn.post url, {}, options[:headers] do |request|
          request.body = options[:data]
        end
      else
        response = conn.get url, {}, options[:headers]
      end
      # set number of available API calls for agents
      if options[:agent_id].present?
        agent = Agent.where(id: options[:agent_id]).first
        agent.update_attributes(rate_limit_remaining: get_rate_limit_remaining(response.headers),
                                rate_limit_reset: get_rate_limit_reset(response.headers),
                                last_response: Time.zone.now)
      end
      # parsing by content type is not reliable, so we check the response format
      if is_json?(response.body)
        JSON.parse(response.body)
      elsif is_xml?(response.body)
        Hash.from_xml(response.body)
      else
        response.body
      end
    rescue *NETWORKABLE_EXCEPTIONS => e
      rescue_faraday_error(url, e, options)
    end

    def set_request_headers(url, options)
      options[:headers] ||= {}
      options[:headers]['Host'] = URI.parse(url).host

      if options[:content_type].present?
        accept_headers = { "html" => 'text/html; charset=UTF-8',
                           "xml" => 'application/xml',
                           "json" => 'application/json' }
        options[:headers]['Accept'] = accept_headers.fetch(options[:content_type], options[:content_type])
      end

      if options[:bearer].present?
        options[:headers]['Authorization'] = "Bearer #{options[:bearer]}"
      elsif options[:token].present?
        options[:headers]["Authorization"] = "Token token=#{options[:token]}"
      elsif options[:username].present?
        options[:headers]["Authorization"] = ActionController::HttpAuthentication::Basic.encode_credentials(options[:username], options[:password])
      end

      options[:headers]
    end

    def faraday_conn(options = {})
      options[:headers] ||= {}
      options[:headers]['Accept'] ||= "application/json"

      limit = options[:limit] || 10

      Faraday.new do |c|
        c.headers['Accept'] = options[:headers]['Accept']
        c.headers['User-Agent'] = "Lagotto - http://#{ENV['SERVERNAME']}"
        c.use      FaradayMiddleware::FollowRedirects, limit: limit, cookie: :all
        c.request  :multipart
        c.request  :json if options[:headers]['Accept'] == 'application/json'
        c.use      Faraday::Response::RaiseError
        c.adapter  Faraday.default_adapter
      end
    end

    def rescue_faraday_error(url, error, options={})
      if error.is_a?(Faraday::ResourceNotFound)
        not_found_error(url, error, options)
      else
        details = nil
        headers = {}

        if error.is_a?(Faraday::Error::TimeoutError)
          status = 408
        elsif error.respond_to?('status')
{'fraction_non_alphanumeric': 0.12626656274356976, 'fraction_numerical': 0.002494154325798909, 'mean_word_length': 2.723737666860128, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7639653', 'n_tokens_mistral': 1865, 'n_tokens_neox': 1808, 'n_words': 374}