<commit_before>require 'net/http'
module Smev
  class Downloader

    def initialize dir_path, max_attempt = 3
      @dir_path = dir_path
      @max_attempt = max_attempt
    end

    def run wsdl_url
      FileUtils.mkdir_p @dir_path, :mode => 0777 unless File.exist? @dir_path
      @imports = {}
      get_remote wsdl_url do |body|
        process4xsd body, 'xsd', wsdl_url
        File.open( File.join(@dir_path, "wsdl"), 'w' ){|f| f.write body }
      end
      true
    end

    class AttemptException < SmevException
    end

  private

    def process4xsd text, name, root_url
      text.scan(/(schemaLocation=\"([^\"]+)\")/i).each_with_index do |match, i|
        unless @imports.include? match.last
          get_remote match.last, root_url do |body|
            child = "#{name}-#{i}"
            @imports[match.last] = child
            process4xsd body, child, root_url
            File.open( File.join(@dir_path, child), 'w' ){|f| f.write body } 
          end
        end
        text.sub! match.first, "schemaLocation=\"#{@imports[match.last]}\""       
      end
    end

    def get_remote url, root_url = 'http:/', attempt = 0
      url = URI::join(root_url, url) if url.is_a?(String) and not url.strip.start_with? "http://"
      raise AttemptException.new("Failed download #{url}") if attempt > @max_attempt
      puts "[GET] " + url.to_s
      res = Net::HTTP.get_response URI( url ) 
      if res.is_a?(Net::HTTPSuccess)
        body = res.body.force_encoding("UTF-8")
        if body.index("schema") and block_given?
          return yield(body)
        end
      end
      get_remote url, root_url, attempt+1
    end

  end
end<commit_msg>continue download if 301 get<commit_after>require 'net/http'
module Smev
  class Downloader

    def initialize dir_path, max_attempt = 3
      @dir_path = dir_path
      @max_attempt = max_attempt
    end

    def run wsdl_url
      FileUtils.mkdir_p @dir_path, :mode => 0777 unless File.exist? @dir_path
      @imports = {}
      get_remote wsdl_url do |body|
        process4xsd body, 'xsd', wsdl_url
        File.open( File.join(@dir_path, "wsdl"), 'w' ){|f| f.write body }
      end
      true
    end

    class AttemptException < SmevException
    end

  private

    def process4xsd text, name, root_url
      text.scan(/(schemaLocation=\"([^\"]+)\")/i).each_with_index do |match, i|
        unless @imports.include? match.last
          get_remote match.last, root_url do |body|
            child = "#{name}-#{i}"
            @imports[match.last] = child
            process4xsd body, child, root_url
            File.open( File.join(@dir_path, child), 'w' ){|f| f.write body } 
          end
        end
        text.sub! match.first, "schemaLocation=\"#{@imports[match.last]}\""       
      end
    end

    def get_remote url, root_url = 'http:/', attempt = 0
      url = URI::join(root_url, url) if url.is_a?(String) and not url.strip.start_with? "http://"
      raise AttemptException.new("Failed download #{url}") if attempt > @max_attempt
      puts "[GET] " + url.to_s
      res = Net::HTTP.get_response URI( url ) 
      if res.is_a?(Net::HTTPSuccess)
        body = res.body.force_encoding("UTF-8")
        if body.index("schema") and block_given?
          return yield(body)
        end
      elsif res.code == "301"
        return get_remote(res.header['location'], attempt)
      end
      get_remote url, root_url, attempt+1
    end

  end
end
{'fraction_non_alphanumeric': 0.125, 'fraction_numerical': 0.008158508158508158, 'mean_word_length': 2.3590998043052838, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7282887', 'n_tokens_mistral': 1172, 'n_tokens_neox': 1118, 'n_words': 285}