<commit_before>module Crusher
  
  class HttpLoadGenerator < LoadGenerator
    
    
    def initialize(options)
      super(options)
      @cookies = ::WebAgent::CookieManager.new 
    end
    
    def request(method, path, content = {}, &block)

      uri = URI.parse(path)

      post_args = content.keys.map do |key| 
        content[key] ? "#{URI.escape(key)}=#{URI.escape(content[key])}" : nil
      end

      post_args = post_args.compact.join("&")

      options = {
        :verb => method,
        :host => uri.host,
        :port => uri.port,
        :request => uri.path,
        :query_string => uri.query,
        :content => post_args,
        :contenttype => 'application/x-www-form-urlencoded',
        :cookie => @cookies.find(uri)
      }

      puts options.inspect

      start_time = Time.now
      EM::Protocols::HttpClient.request(options).callback do |response|
        duration = Time.now - start_time
        
        response[:headers].each do |header|
          
          matcher = /Set-Cookie:\s*(.*)/i.match(header)
          
          if matcher
            puts "Found cookie-like header: #{header}"
            @cookies.parse(matcher[1], uri)
          end
        end
        
        block.call(response, duration) if block
      end

    end
    
    
  end
  
end<commit_msg>Fixed cookie handling and added verb-specific handling of HTTP arguments<commit_after>module Crusher
  
  class HttpLoadGenerator < LoadGenerator
    
    
    def initialize(options)
      super(options)
      @cookies = ::WebAgent::CookieManager.new 
    end
    
    def request(method, path, content = {}, &block)

      uri = URI.parse(path)

      http_args = content.keys.map do |key| 
        content[key] ? "#{URI.escape(key.to_s)}=#{URI.escape(content[key].to_s)}" : nil
      end

      

      qs, content = if method == :get
        [((uri.query || '').split('&') + http_args.compact).join("&"), nil]
      else
        [uri.query, http_args.join("&")] 
      end

      options = {
        :verb => method,
        :host => uri.host,
        :port => uri.port,
        :request => uri.path,
        :query_string => qs,
        :content => content,
        :contenttype => 'application/x-www-form-urlencoded',
        :cookie => @cookies.find(uri)
      }

      start_time = Time.now
      EM::Protocols::HttpClient.request(options).callback do |response|
        duration = Time.now - start_time
        
        response[:headers].each do |header|
          matcher = /Set-Cookie:\s*(.*)/i.match(header)
          @cookies.parse(matcher[1], uri) if matcher
        end
        
        block.call(response, duration) if block
      end

    end
    
    
  end
  
end
{'fraction_non_alphanumeric': 0.13148148148148148, 'fraction_numerical': 0.0007407407407407407, 'mean_word_length': 1.923160173160173, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6181990', 'n_tokens_mistral': 837, 'n_tokens_neox': 817, 'n_words': 169}