<commit_before># To change this template, choose Tools | Templates
# and open the template in the editor.

require 'openssl'
module Jerm
  OpenSSL::SSL::VERIFY_PEER = OpenSSL::SSL::VERIFY_NONE
  
  class HttpDownloader
    def get_remote_data url, username=nil, password=nil, type=nil
      return basic_auth url, username, password
    end

    private

    #handles fetching data using basic authentication. Handles http and https.
    #returns a hash that contains the following:
    # :data=> the data
    # :content_type=> the content_type
    # :filename => the filename
    #
    # throws an Exception if anything goes wrong.
    def basic_auth url, username,password      
      begin
        open(URI.encode(url),:http_basic_authentication=>[username, password]) do |f|
          #FIXME: need to handle full range of 2xx sucess responses, in particular where the response is only partial
          if f.status[0] == "200"                    
            return {:data=>f.read,:content_type=>f.content_type,:filename=>f.base_uri.path.split('/').last}
          else
            raise Exception.new("Problem fetching data from remote site - response code #{thing.status[0]}, url: #{url}")
          end
        end        
      rescue OpenURI::HTTPError => error
        raise Exception.new("Problem fetching data from remote site - response code #{error.io.status[0]}, url:#{url}")
      end
    end
    
  end
end<commit_msg>fixed issue with jerm downloader where previsouly URI.encoded urls were being re-encoded<commit_after># To change this template, choose Tools | Templates
# and open the template in the editor.

require 'openssl'
module Jerm
  OpenSSL::SSL::VERIFY_PEER = OpenSSL::SSL::VERIFY_NONE
  
  class HttpDownloader
    def get_remote_data url, username=nil, password=nil, type=nil
      return basic_auth url, username, password
    end

    private

    #handles fetching data using basic authentication. Handles http and https.
    #returns a hash that contains the following:
    # :data=> the data
    # :content_type=> the content_type
    # :filename => the filename
    #
    # throws an Exception if anything goes wrong.
    def basic_auth url, username,password

      #This block is to ensure that only urls are encoded if they need it.
      #This is to prevent already encoded urls being re-encoded, which can lead to % being replaced with %25.
      begin
        URI.parse(url)
      rescue URI::InvalidURIError
        url=URI.encode(url)
      end
      begin
        open(url,:http_basic_authentication=>[username, password]) do |f|
          #FIXME: need to handle full range of 2xx sucess responses, in particular where the response is only partial
          if f.status[0] == "200"                    
            return {:data=>f.read,:content_type=>f.content_type,:filename=>f.base_uri.path.split('/').last}
          else
            raise Exception.new("Problem fetching data from remote site - response code #{thing.status[0]}, url: #{url}")
          end
        end        
      rescue OpenURI::HTTPError => error
        raise Exception.new("Problem fetching data from remote site - response code #{error.io.status[0]}, url:#{url}")
      end
    end
    
  end
end
{'fraction_non_alphanumeric': 0.08944099378881988, 'fraction_numerical': 0.004968944099378882, 'mean_word_length': 2.7849588719153937, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3715201', 'n_tokens_mistral': 901, 'n_tokens_neox': 889, 'n_words': 315}