<commit_before>require 'net/http'
require 'uri'
require 'pipeline/pipe'

module Pipeline
  class HttpPipe < Pipe
    def initialize
      super('HTTP', 1, 1)
      @config[:path] = '/tmp/hclerk/'
      @config[:ensure_safe_filename] = 'yes'
    end
    
    def check_before_work
      @path = @config[:path]
      @uri = @source.items[0]
      
      if not File.exists?(@path)
        log "creating directory: #{@path}"
        Dir.mkdir @path
      end
      
      return :ok
    end
    
    def work
      return load(@uri, @path)
    end
    
    def check_after_work
      if not File.exists?(@save_to)
        return :fail
      end
      
      @target.add @save_to
      
      return :ok
    end
    
    
    def load(uri, path)
      log "loading uri #{uri}"
      response = Net::HTTP.get_response(URI(uri))
      
      case response
      when Net::HTTPSuccess then
        log "content length #{response.content_length}"
        parse_save_to uri, path
        save_file response
        return :ok
      when Net::HTTPRedirection then
        redir = response['location']
        log "redirection #{redir}"
        load redir, path
        return :ok
      else
        log "error: #{response.value}"
        return :fail
      end
    end
    
    def parse_save_to(uri, path)
      filename = @config[:filename]
      if filename.nil?
        parsed_uri = URI.parse uri
        filename = File.basename parsed_uri.path
      end
      filename = URI.unescape(filename)
      
      if @config[:ensure_safe_filename] == 'yes' or @config[:ensure_safe_filename] == 'y'
        filename.gsub! /'/, '_'
        filename.gsub! /\s/, '_'
        filename.gsub! /-/, '_'
        filename.gsub! /_+/, '_'
      end
      
      @save_to = File.join path, filename
    end
    
    def save_file(response)
      File.open(@save_to, 'w') do |f|
        log "writing file #{@save_to}"
        f.write response.body
      end
    end
    
    def log_headers(response)
      response.each do |hkey, hval|
        log "header: #{hkey}=#{hval}"
      end
    end
    
  end
end<commit_msg>Http: write file in binary mode.<commit_after>require 'net/http'
require 'uri'
require 'pipeline/pipe'

module Pipeline
  class HttpPipe < Pipe
    def initialize
      super('HTTP', 1, 1)
      @config[:path] = '/tmp/hclerk/'
      @config[:ensure_safe_filename] = 'yes'
    end
    
    def check_before_work
      @path = @config[:path]
      @uri = @source.items[0]
      
      if not File.exists?(@path)
        log "creating directory: #{@path}"
        Dir.mkdir @path
      end
      
      return :ok
    end
    
    def work
      return load(@uri, @path)
    end
    
    def check_after_work
      if not File.exists?(@save_to)
        return :fail
      end
      
      @target.add @save_to
      
      return :ok
    end
    
    
    def load(uri, path)
      log "loading uri #{uri}"
      response = Net::HTTP.get_response(URI(uri))
      
      case response
      when Net::HTTPSuccess then
        log "content length #{response.content_length}"
        parse_save_to uri, path
        save_file response
        return :ok
      when Net::HTTPRedirection then
        redir = response['location']
        log "redirection #{redir}"
        load redir, path
        return :ok
      else
        log "error: #{response.value}"
        return :fail
      end
    end
    
    def parse_save_to(uri, path)
      filename = @config[:filename]
      if filename.nil?
        parsed_uri = URI.parse uri
        filename = File.basename parsed_uri.path
      end
      filename = URI.unescape(filename)
      
      if @config[:ensure_safe_filename] == 'yes' or @config[:ensure_safe_filename] == 'y'
        filename.gsub! /'/, '_'
        filename.gsub! /\s/, '_'
        filename.gsub! /-/, '_'
        filename.gsub! /_+/, '_'
      end
      
      @save_to = File.join path, filename
    end
    
    def save_file(response)
      File.open(@save_to, 'wb') do |f|
        log "writing file #{@save_to}"
        f.write response.body
      end
    end
    
    def log_headers(response)
      response.each do |hkey, hval|
        log "header: #{hkey}=#{hval}"
      end
    end
    
  end
end
{'fraction_non_alphanumeric': 0.11021377672209026, 'fraction_numerical': 0.0014251781472684087, 'mean_word_length': 1.8054630246502332, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1429243', 'n_tokens_mistral': 1341, 'n_tokens_neox': 1344, 'n_words': 334}