<commit_before>#
# = net/http.rb
#
# Copyright (c) 1999-2007 <NAME>
# Copyright (c) 1999-2007 Minero Aoki
# Copyright (c) 2001 <NAME>
# 
# Written and maintained by <NAME> <<EMAIL>>.
# HTTPS support added by GOTOU Yuuzou <<EMAIL>>.
#
# This file is derived from "http-access.rb".
#
# Documented by Minero Aoki; converted to RDoc by <NAME>.
# 
# This program is free software. You can re-distribute and/or
# modify this program under the same terms of ruby itself ---
# Ruby Distribution License or GNU General Public License.
#
# See Net::HTTP for an overview and examples. 
# 
# NOTE: You can find Japanese version of this document here:
# http://www.ruby-lang.org/ja/man/html/net_http.html
# 
#--
# $Id: http.rb 30571 2011-01-16 12:35:11Z yugui $
#++

require 'net/protocol'
require 'uri'

module Net   #:nodoc:

  # :stopdoc:
  class HTTPBadResponse < StandardError; end
  class HTTPHeaderSyntaxError < StandardError; end
  # :startdoc:

  # == What Is This Library?
  # 
  # This library provides your program functions to access WWW
  # documents via HTTP, Hyper Text Transfer Protocol version 1.1.
  # For details of HTTP, refer to [RFC2616]
  # (http://www.ietf.org/rfc/rfc2616.txt).
  # 
  # == Examples
  # 
  # === Getting Document From WWW Server
  # 
  # Example #1: Simple GET+print
  # 
  #     require 'net/http'
  #     Net::HTTP.get_print 'www.example.com', '/index.html'
  # 
  # Example #2: Simple GET+print by URL
  # 
  #     require 'net/http'
  #     require 'uri'
  #     Net::HTTP.get_print URI.parse('http://www.example.com/index.html')
  # 
  # Example #3: More generic GET+print
  # 
  #     require 'net/http'
  #     require 'uri'
  #
  #     url = URI.parse('http://www.example.com/index.html')
  #     res = Net::HTTP.start(url.host, url.port) {|http|
  #       http.get('/index.html')
  #     }
  #     puts res.body
  #
  # Example #4: More generic GET+print
  # 
  #     require 'net/http'
  #
  #     url = URI.parse('http://www.example.com/index.html')
  #     req = Net::HTTP::Get.new(url.path)
  #     res = Net::HTTP.start(url.host, url.port) {|http|
  #       http.request(req)
  #     }
  #     puts res.body
  # 
  # === Posting Form Data
  # 
  #     require 'net/http'
  #     require 'uri'
  #
  #     #1: Simple POST
  #     res = Net::HTTP.post_form(URI.parse('http://www.example.com/search.cgi'),
  #                               {'q' => 'ruby', 'max' => '50'})
  #     puts res.body
  #
  #     #2: POST with basic authentication
  #     res = Net::HTTP.post_form(URI.parse('http://jack:pass@www.example.com/todo.cgi'),
  #                                         {'from' => '2005-01-01',
  #                                          'to' => '2005-03-31'})
  #     puts res.body
  #
  #     #3: Detailed control
  #     url = URI.parse('http://www.example.com/todo.cgi')
  #     req = Net::HTTP::Post.new(url.path)
  #     req.basic_auth 'jack', '<PASSWORD>'
  #     req.set_form_data({'from' => '2005-01-01', 'to' => '2005-03-31'}, ';')
  #     res = Net::HTTP.new(url.host, url.port).start {|http| http.request(req) }
  #     case res
  #     when Net::HTTPSuccess, Net::HTTPRedirection
  #       # OK
  #     else
  #       res.error!
  #     end
  #
  #     #4: Multiple values
  #     res = Net::HTTP.post_form(URI.parse('http://www.example.com/search.cgi'),
  #                               {'q' => ['ruby', 'perl'], 'max' => '50'})
  #     puts res.body
  # 
  # === Accessing via Proxy
  # 
  # Net::HTTP.Proxy creates http proxy class. It has same
  # methods of Net::HTTP but its instances always connect to
  # proxy, instead of given host.
  # 
  #     require 'net/http'
  # 
  #     proxy_addr = 'your.proxy.host'
  #     proxy_port = 8080
  #             :
  #     Net::HTTP::Proxy(proxy_addr, proxy_port).start('www.example.com') {|http|
  #       # always connect to your.proxy.addr:8080
  #             :
  #     }
  # 
  # Since Net::HTTP.Proxy returns Net::HTTP itself when proxy_addr is nil,
  # there's no need to change code if there's proxy or not.
  # 
  # There are two additional parameters in Net::HTTP.Proxy which allow to
  # specify proxy user name and password:
  # 
  #     Net::HTTP::Proxy(proxy_addr, proxy_port, proxy_user = nil, proxy_pass = nil)
  # 
  # You may use them to work with authorization-enabled proxies:
  # 
  #     require 'net/http'
  #     require 'uri'
  #     
  #     proxy_host = 'your.proxy.host'
  #     proxy_port = 8080
  #     uri = URI.parse(ENV['http_proxy'])
  #     proxy_user, proxy_pass = uri.userinfo.split(/:/) if uri.userinfo
  #     Net::HTTP::Proxy(proxy_host, proxy_port,
  #                      proxy_user, proxy_pass).start('www.example.com') {|http|
  #       # always connect to your.proxy.addr:8080 using specified username and password
  #             :
  #     }
  #
  # Note that net/http never rely on HTTP_PROXY environment variable.
  # If you want to use proxy, set it explicitly.
  # 
  # === Following Redirection
  # 
  #     require 'net/http'
  #     require 'uri'
  # 
  #     def fetch(uri_str, limit = 10)
  #       # You should choose better exception. 
  #       raise ArgumentError, 'HTTP redirect too deep' if limit == 0
  # 
  #       response = Net::HTTP.get_response(URI.parse(uri_str))
  #       case response
  #       when Net::HTTPSuccess     then response
  #       when Net::HTTPRedirection then fetch(response['location'], limit - 1)
  #       else
  #         response.error!
  #       end
  #     end
  # 
  #     print fetch('http://www.ruby-lang.org')
  # 
  # Net::HTTPSuccess and Net::HTTPRedirection is a HTTPResponse class.
  # All HTTPResponse objects belong to its own response class which
  # indicate HTTP result status. For details of response classes,
  # see section "HTTP Response Classes".
  # 
  # === Basic Authentication
  # 
  #     require 'net/http'
  # 
  #     Net::HTTP.start('www.example.com') {|http|
  #       req = Net::HTTP::Get.new('/secret-page.html')
  #       req.basic_auth 'account', 'password'
  #       response = http.request(req)
  #       print response.body
  #     }
  # 
  # === HTTP Request Classes
  #
  # Here is HTTP request class hierarchy.
  #
  #   Net::HTTPRequest
  #       Net::HTTP::Get
  #       Net::HTTP::Head
  #       Net::HTTP::Post
  #       Net::HTTP::Put
  #       Net::HTTP::Proppatch
  #       Net::HTTP::Lock
  #       Net::HTTP::Unlock
  #       Net::HTTP::Options
  #       Net::HTTP::Propfind
  #       Net::HTTP::Delete
  #       Net::HTTP::Move
  #       Net::HTTP::Copy
  #       Net::HTTP::Mkcol
  #       Net::HTTP::Trace
  #
  # === HTTP Response Classes
  #
  # Here is HTTP response class hierarchy.
  # All classes are defined in Net module.
  #
  #   HTTPResponse
  #       HTTPUnknownResponse
  #       HTTPInformation                    # 1xx
  #           HTTPContinue                       # 100
  #           HTTPSwitchProtocol                 # 101
  #       HTTPSuccess                        # 2xx
  #           HTTPOK                             # 200
  #           HTTPCreated                        # 201
  #           HTTPAccepted                       # 202
  #           HTTPNonAuthoritativeInformation    # 203
  #           HTTPNoContent                      # 204
  #           HTTPResetContent                   # 205
  #           HTTPPartialContent                 # 206
  #       HTTPRedirection                    # 3xx
  #           HTTPMultipleChoice                 # 300
  #           HTTPMovedPermanently               # 301
  #           HTTPFound                          # 302
  #           HTTPSeeOther                       # 303
  #           HTTPNotModified                    # 304
  #           HTTPUseProxy                       # 305
  #           HTTPTemporaryRedirect              # 307
  #       HTTPClientError                    # 4xx
  #           HTTPBadRequest                     # 400
  #           HTTPUnauthorized                   # 401
  #           HTTPPaymentRequired                # 402
  #           HTTPForbidden                      # 403
  #           HTTPNotFound                       # 404
  #           HTTPMethodNotAllowed               # 405
  #           HTTPNotAcceptable                  # 406
  #           HTTPProxyAuthenticationRequired    # 407
  #           HTTPRequestTimeOut                 # 408
  #           HTTPConflict                       # 409
  #           HTTPGone                           # 410
  #           HTTPLengthRequired                 # 411
  #           HTTPPreconditionFailed             # 412
  #           HTTPRequestEntityTooLarge          # 413
  #           HTTPRequestURITooLong              # 414
  #           HTTPUnsupportedMediaType           # 415
  #           HTTPRequestedRangeNotSatisfiable   # 416
  #           HTTPExpectationFailed              # 417
  #       HTTPServerError                    # 5xx
  #           HTTPInternalServerError            # 500
  #           HTTPNotImplemented                 # 501
  #           HTTPBadGateway                     # 502
  #           HTTPServiceUnavailable             # 503
  #           HTTPGatewayTimeOut                 # 504
  #           HTTPVersionNotSupported            # 505
  # 
  # == Switching Net::HTTP versions
  # 
  # You can use net/http.rb 1.1 features (bundled with Ruby 1.6)
  # by calling HTTP.version_1_1. Calling Net::HTTP.version_1_2
  # allows you to use 1.2 features again.
  # 
  #     # example
  #     Net::HTTP.start {|http1| ...(http1 has 1.2 features)... }
  # 
  #     Net::HTTP.version_1_1
  #     Net::HTTP.start {|http2| ...(http2 has 1.1 features)... }
  # 
  #     Net::HTTP.version_1_2
  #     Net::HTTP.start {|http3| ...(http3 has 1.2 features)... }
  # 
  # This function is NOT thread-safe.
  #
  class HTTP < Protocol

    # :stopdoc:
    Revision = %q$Revision: 30571 $.split[1]
    HTTPVersion = '1.1'
    @newimpl = true
    begin
      if defined? RHO_ME
        HAVE_ZLIB=false      
      else  
        require 'zlib'
        require 'stringio'  #for our purposes (unpacking gzip) lump these together
        HAVE_ZLIB=true
      end    
    rescue LoadError
      HAVE_ZLIB=false
    end
    # :startdoc:

    # Turns on net/http 1.2 (ruby 1.8) features.
    # Defaults to ON in ruby 1.8.
    #
    # I strongly recommend to call this method always.
    #
    #   require 'net/http'
    #   Net::HTTP.version_1_2
    #
    def HTTP.version_1_2
      @newimpl = true
    end

    # Turns on net/http 1.1 (ruby 1.6) features.
    # Defaults to OFF in ruby 1.8.
    def HTTP.version_1_1
      @newimpl = false
    end

    # true if net/http is in version 1.2 mode.
    # Defaults to true.
    def HTTP.version_1_2?
      @newimpl
    end

    # true if net/http is in version 1.1 compatible mode.
    # Defaults to true.
    def HTTP.version_1_1?
      not @newimpl
    end

    class << HTTP
      alias is_version_1_1? version_1_1?   #:nodoc:
      alias is_version_1_2? version_1_2?   #:nodoc:
    end

    #
    # short cut methods
    #

    #
    # Get body from target and output it to +$stdout+.  The
    # target can either be specified as (+uri+), or as
    # (+host+, +path+, +port+ = 80); so: 
    #
    #    Net::HTTP.get_print URI.parse('http://www.example.com/index.html')
    #
    # or:
    #
    #    Net::HTTP.get_print 'www.example.com', '/index.html'
    #
    def HTTP.get_print(uri_or_host, path = nil, port = nil)
      get_response(uri_or_host, path, port) {|res|
        res.read_body do |chunk|
          $stdout.print chunk
        end
      }
      nil
    end

    # Send a GET request to the target and return the response
    # as a string.  The target can either be specified as
    # (+uri+), or as (+host+, +path+, +port+ = 80); so:
    # 
    #    print Net::HTTP.get(URI.parse('http://www.example.com/index.html'))
    #
    # or:
    #
    #    print Net::HTTP.get('www.example.com', '/index.html')
    #
    def HTTP.get(uri_or_host, path = nil, port = nil)
      get_response(uri_or_host, path, port).body
    end

    # Send a GET request to the target and return the response
    # as a Net::HTTPResponse object.  The target can either be specified as
    # (+uri+), or as (+host+, +path+, +port+ = 80); so:
    # 
    #    res = Net::HTTP.get_response(URI.parse('http://www.example.com/index.html'))
    #    print res.body
    #
    # or:
    #
    #    res = Net::HTTP.get_response('www.example.com', '/index.html')
    #    print res.body
    #
    def HTTP.get_response(uri_or_host, path = nil, port = nil, &block)
      if path
        host = uri_or_host
        new(host, port || HTTP.default_port).start {|http|
          return http.request_get(path, &block)
        }
      else
        uri = uri_or_host
        new(uri.host, uri.port).start {|http|
          return http.request_get(uri.request_uri, &block)
        }
      end
    end

    # Posts HTML form data to the +URL+.
    # Form data must be represented as a Hash of String to String, e.g:
    #
    #   { "cmd" => "search", "q" => "ruby", "max" => "50" }
    #
    # This method also does Basic Authentication iff +URL+.user exists.
    #
    # Example:
    #
    #   require 'net/http'
    #   require 'uri'
    #
    #   HTTP.post_form URI.parse('http://www.example.com/search.cgi'),
    #                  { "q" => "ruby", "max" => "50" }
    #
    def HTTP.post_form(url, params)
      req = Post.new(url.path)
      req.form_data = params
      req.basic_auth url.user, url.password if url.user
      new(url.host, url.port).start {|http|
        http.request(req)
      }
    end

    #
    # HTTP session management
    #

    # The default port to use for HTTP requests; defaults to 80.
    def HTTP.default_port
      http_default_port()
    end

    # The default port to use for HTTP requests; defaults to 80.
    def HTTP.http_default_port
      80
    end

    # The default port to use for HTTPS requests; defaults to 443.
    def HTTP.https_default_port
      443
    end

    def HTTP.socket_type   #:nodoc: obsolete
      BufferedIO
    end

    # call-seq:
    #   HTTP.start(address, port, p_addr, p_port, p_user, p_pass, &block)
    #   HTTP.start(address, port=nil, p_addr=nil, p_port=nil, p_user=nil, p_pass=nil, opt, &block)
    #
    # creates a new Net::HTTP object and opens its TCP connection and
    # HTTP session.
    #
    # Argments are following:
    # _address_ :: hostname or IP address of the server
    # _port_    :: port of the server
    # _p_addr_  :: address of proxy
    # _p_port_  :: port of proxy
    # _p_user_  :: user of proxy
    # _p_pass_  :: pass of proxy
    # _opt_     :: optional hash
    #
    # _opt_ sets following values by its accessor.
    # The keys are ca_file, ca_path, cert, cert_store, ciphers,
    # close_on_empty_response, key, open_timeout, read_timeout, ssl_timeout,
    # ssl_version, use_ssl, verify_callback, verify_depth and verify_mode.
    # If you set :use_ssl as true, you can use https and default value of
    # verify_mode is set as OpenSSL::SSL::VERIFY_PEER.
    #
    # If the optional block is given, the newly
    # created Net::HTTP object is passed to it and closed when the
    # block finishes.  In this case, the return value of this method
    # is the return value of the block.  If no block is given, the
    # return value of this method is the newly created Net::HTTP object
    # itself, and the caller is responsible for closing it upon completion.
    def HTTP.start(address, *arg, &block) # :yield: +http+
      arg.pop if opt = Hash.try_convert(arg[-1])
      port, p_addr, p_port, p_user, p_pass = *arg
      port = https_default_port if !port && opt && opt[:use_ssl]
      http = new(address, port, p_addr, p_port, p_user, p_pass)

      #if opt
      #  opt = {verify_mode: OpenSSL::SSL::VERIFY_PEER}.update(opt) if opt[:use_ssl]
      #  http.methods.grep(/\A(\w+)=\z/) do |meth|
      #    key = $1.to_sym
      #    opt.key?(key) or next
      #    http.__send__(meth, opt[key])
      #  end
     # end

      http.start(&block)
    end

    class << HTTP
      alias newobj new
    end

    # Creates a new Net::HTTP object.
    # If +proxy_addr+ is given, creates an Net::HTTP object with proxy support.
    # This method does not open the TCP connection.
    def HTTP.new(address, port = nil, p_addr = nil, p_port = nil, p_user = nil, p_pass = nil)
      h = Proxy(p_addr, p_port, p_user, p_pass).newobj(address, port)
      h.instance_eval {
        @newimpl = ::Net::HTTP.version_1_2?
      }
      h
    end

    # Creates a new Net::HTTP object for the specified +address+.
    # This method does not open the TCP connection.
    def initialize(address, port = nil)
      @address = address
      @port    = (port || HTTP.default_port)
      @curr_http_version = HTTPVersion
      @no_keepalive_server = false
      @close_on_empty_response = false
      @socket  = nil
      @started = false
      @open_timeout = nil
      @read_timeout = 60
      @debug_output = nil
      @use_ssl = false
      @ssl_context = nil
      @enable_post_connection_check = true
      @compression = nil
      @sspi_enabled = false
      if defined?(SSL_ATTRIBUTES)
        SSL_ATTRIBUTES.each do |name|
          instance_variable_set "@#{name}", nil
        end
      end
    end

    def inspect
      "#<#{self.class} #{@address}:#{@port} open=#{started?}>"
    end

    # *WARNING* This method causes serious security hole.
    # Never use this method in production code.
    #
    # Set an output stream for debugging.
    #
    #   http = Net::HTTP.new
    #   http.set_debug_output $stderr
    #   http.start { .... }
    #
    def set_debug_output(output)
      warn 'Net::HTTP#set_debug_output called after HTTP started' if started?
      @debug_output = output
    end

    # The host name to connect to.
    attr_reader :address

    # The port number to connect to.
    attr_reader :port

    # Seconds to wait until connection is opened.
    # If the HTTP object cannot open a connection in this many seconds,
    # it raises a TimeoutError exception.
    attr_accessor :open_timeout

    # Seconds to wait until reading one block (by one read(2) call).
    # If the HTTP object cannot open a connection in this many seconds,
    # it raises a TimeoutError exception.
    attr_reader :read_timeout

    # Setter for the read_timeout attribute.
    def read_timeout=(sec)
      @socket.read_timeout = sec if @socket
      @read_timeout = sec
    end

    # returns true if the HTTP session is started.
    def started?
      @started
    end

    alias active? started?   #:nodoc: obsolete

    attr_accessor :close_on_empty_response

    # returns true if use SSL/TLS with HTTP.
    def use_ssl?
      false   # redefined in net/https
    end

    # Opens TCP connection and HTTP session.
    # 
    # When this method is called with block, gives a HTTP object
    # to the block and closes the TCP connection / HTTP session
    # after the block executed.
    #
    # When called with a block, returns the return value of the
    # block; otherwise, returns self.
    #
    def start  # :yield: http
      raise IOError, 'HTTP session already opened' if @started
      if block_given?
        begin
          do_start
          return yield(self)
        ensure
          do_finish
        end
      end
      do_start
      self
    end

    def do_start
      connect
      @started = true
    end
    private :do_start

    def connect
      platform = System::get_property('platform')
      if !(use_ssl? and platform == 'Blackberry')
        D "opening connection to #{conn_address()}..."
        s = timeout(@open_timeout) { TCPSocket.open(conn_address(), conn_port()) }
        D "opened"
      end
      if use_ssl?
        ssl_parameters = Hash.new
        iv_list = instance_variables
        SSL_ATTRIBUTES.each do |name|
          ivname = "@#{name}".intern
          if iv_list.include?(ivname) and
             value = instance_variable_get(ivname)
            ssl_parameters[name] = value
          end
        end
        if platform != 'Blackberry'
          @ssl_context = OpenSSL::SSL::SSLContext.new
          @ssl_context.set_params(ssl_parameters)
          s = OpenSSL::SSL::SSLSocket.new(s, @ssl_context)
          s.sync_close = true
        else
          s = SSLSocket.open(conn_address(), conn_port())
        end
      end
      @socket = BufferedIO.new(s)
      @socket.read_timeout = @read_timeout
      @socket.debug_output = @debug_output
      if use_ssl?
        begin
          if proxy?
            @socket.writeline sprintf('CONNECT %s:%s HTTP/%s',
                                      @address, @port, HTTPVersion)
            @socket.writeline "Host: #{@address}:#{@port}"
            if proxy_user
              credential = ["#{proxy_user}:#{proxy_pass}"].pack('m')
              credential.delete!("\r\n")
              @socket.writeline "Proxy-Authorization: Basic #{credential}"
            end
            @socket.writeline ''
            HTTPResponse.read_new(@socket).value
          end
          timeout(@open_timeout) { s.connect }
          if @ssl_context.verify_mode != OpenSSL::SSL::VERIFY_NONE
            s.post_connection_check(@address)
          end
        rescue => exception
          D "Conn close because of connect error #{exception}"
          @socket.close if @socket and not @socket.closed?
          raise exception
        end
      end
      on_connect
    end
    
    private :connect

    def on_connect
    end
    private :on_connect
<commit_msg>bb: fix net-http build<commit_after>#
# = net/http.rb
#
# Copyright (c) 1999-2007 <NAME>
# Copyright (c) 1999-2007 <NAME>
# Copyright (c) 2001 <NAME>
# 
# Written and maintained by <NAME> <<EMAIL>>.
# HTTPS support added by GOTOU Yuuzou <<EMAIL>>.
#
# This file is derived from "http-access.rb".
#
# Documented by Minero Aoki; converted to RDoc by <NAME>.
# 
# This program is free software. You can re-distribute and/or
# modify this program under the same terms of ruby itself ---
# Ruby Distribution License or GNU General Public License.
#
# See Net::HTTP for an overview and examples. 
# 
# NOTE: You can find Japanese version of this document here:
# http://www.ruby-lang.org/ja/man/html/net_http.html
# 
#--
# $Id: http.rb 30571 2011-01-16 12:35:11Z yugui $
#++

require 'net/protocol'
require 'uri'

module Net   #:nodoc:

  # :stopdoc:
  class HTTPBadResponse < StandardError; end
  class HTTPHeaderSyntaxError < StandardError; end
  # :startdoc:

  # == What Is This Library?
  # 
  # This library provides your program functions to access WWW
  # documents via HTTP, Hyper Text Transfer Protocol version 1.1.
  # For details of HTTP, refer to [RFC2616]
  # (http://www.ietf.org/rfc/rfc2616.txt).
  # 
  # == Examples
  # 
  # === Getting Document From WWW Server
  # 
  # Example #1: Simple GET+print
  # 
  #     require 'net/http'
  #     Net::HTTP.get_print 'www.example.com', '/index.html'
  # 
  # Example #2: Simple GET+print by URL
  # 
  #     require 'net/http'
  #     require 'uri'
  #     Net::HTTP.get_print URI.parse('http://www.example.com/index.html')
  # 
  # Example #3: More generic GET+print
  # 
  #     require 'net/http'
  #     require 'uri'
  #
  #     url = URI.parse('http://www.example.com/index.html')
  #     res = Net::HTTP.start(url.host, url.port) {|http|
  #       http.get('/index.html')
  #     }
  #     puts res.body
  #
  # Example #4: More generic GET+print
  # 
  #     require 'net/http'
  #
  #     url = URI.parse('http://www.example.com/index.html')
  #     req = Net::HTTP::Get.new(url.path)
  #     res = Net::HTTP.start(url.host, url.port) {|http|
  #       http.request(req)
  #     }
  #     puts res.body
  # 
  # === Posting Form Data
  # 
  #     require 'net/http'
  #     require 'uri'
  #
  #     #1: Simple POST
  #     res = Net::HTTP.post_form(URI.parse('http://www.example.com/search.cgi'),
  #                               {'q' => 'ruby', 'max' => '50'})
  #     puts res.body
  #
  #     #2: POST with basic authentication
  #     res = Net::HTTP.post_form(URI.parse('http://jack:pass@www.example.com/todo.cgi'),
  #                                         {'from' => '2005-01-01',
  #                                          'to' => '2005-03-31'})
  #     puts res.body
  #
  #     #3: Detailed control
  #     url = URI.parse('http://www.example.com/todo.cgi')
  #     req = Net::HTTP::Post.new(url.path)
  #     req.basic_auth 'jack', '<PASSWORD>'
  #     req.set_form_data({'from' => '2005-01-01', 'to' => '2005-03-31'}, ';')
  #     res = Net::HTTP.new(url.host, url.port).start {|http| http.request(req) }
  #     case res
  #     when Net::HTTPSuccess, Net::HTTPRedirection
  #       # OK
  #     else
  #       res.error!
  #     end
  #
  #     #4: Multiple values
  #     res = Net::HTTP.post_form(URI.parse('http://www.example.com/search.cgi'),
  #                               {'q' => ['ruby', 'perl'], 'max' => '50'})
  #     puts res.body
  # 
  # === Accessing via Proxy
  # 
  # Net::HTTP.Proxy creates http proxy class. It has same
  # methods of Net::HTTP but its instances always connect to
  # proxy, instead of given host.
  # 
  #     require 'net/http'
  # 
  #     proxy_addr = 'your.proxy.host'
  #     proxy_port = 8080
  #             :
  #     Net::HTTP::Proxy(proxy_addr, proxy_port).start('www.example.com') {|http|
  #       # always connect to your.proxy.addr:8080
  #             :
  #     }
  # 
  # Since Net::HTTP.Proxy returns Net::HTTP itself when proxy_addr is nil,
  # there's no need to change code if there's proxy or not.
  # 
  # There are two additional parameters in Net::HTTP.Proxy which allow to
  # specify proxy user name and password:
  # 
  #     Net::HTTP::Proxy(proxy_addr, proxy_port, proxy_user = nil, proxy_pass = nil)
  # 
  # You may use them to work with authorization-enabled proxies:
  # 
  #     require 'net/http'
  #     require 'uri'
  #     
  #     proxy_host = 'your.proxy.host'
  #     proxy_port = 8080
  #     uri = URI.parse(ENV['http_proxy'])
  #     proxy_user, proxy_pass = uri.userinfo.split(/:/) if uri.userinfo
  #     Net::HTTP::Proxy(proxy_host, proxy_port,
  #                      proxy_user, proxy_pass).start('www.example.com') {|http|
  #       # always connect to your.proxy.addr:8080 using specified username and password
  #             :
  #     }
  #
  # Note that net/http never rely on HTTP_PROXY environment variable.
  # If you want to use proxy, set it explicitly.
  # 
  # === Following Redirection
  # 
  #     require 'net/http'
  #     require 'uri'
  # 
  #     def fetch(uri_str, limit = 10)
  #       # You should choose better exception. 
  #       raise ArgumentError, 'HTTP redirect too deep' if limit == 0
  # 
  #       response = Net::HTTP.get_response(URI.parse(uri_str))
  #       case response
  #       when Net::HTTPSuccess     then response
  #       when Net::HTTPRedirection then fetch(response['location'], limit - 1)
  #       else
  #         response.error!
  #       end
  #     end
  # 
  #     print fetch('http://www.ruby-lang.org')
  # 
  # Net::HTTPSuccess and Net::HTTPRedirection is a HTTPResponse class.
  # All HTTPResponse objects belong to its own response class which
  # indicate HTTP result status. For details of response classes,
  # see section "HTTP Response Classes".
  # 
  # === Basic Authentication
  # 
  #     require 'net/http'
  # 
  #     Net::HTTP.start('www.example.com') {|http|
  #       req = Net::HTTP::Get.new('/secret-page.html')
  #       req.basic_auth 'account', 'password'
  #       response = http.request(req)
  #       print response.body
  #     }
  # 
  # === HTTP Request Classes
  #
  # Here is HTTP request class hierarchy.
  #
  #   Net::HTTPRequest
  #       Net::HTTP::Get
  #       Net::HTTP::Head
  #       Net::HTTP::Post
  #       Net::HTTP::Put
  #       Net::HTTP::Proppatch
  #       Net::HTTP::Lock
  #       Net::HTTP::Unlock
  #       Net::HTTP::Options
  #       Net::HTTP::Propfind
  #       Net::HTTP::Delete
  #       Net::HTTP::Move
  #       Net::HTTP::Copy
  #       Net::HTTP::Mkcol
  #       Net::HTTP::Trace
  #
  # === HTTP Response Classes
  #
  # Here is HTTP response class hierarchy.
  # All classes are defined in Net module.
  #
  #   HTTPResponse
  #       HTTPUnknownResponse
  #       HTTPInformation                    # 1xx
  #           HTTPContinue                       # 100
  #           HTTPSwitchProtocol                 # 101
  #       HTTPSuccess                        # 2xx
  #           HTTPOK                             # 200
  #           HTTPCreated                        # 201
  #           HTTPAccepted                       # 202
  #           HTTPNonAuthoritativeInformation    # 203
  #           HTTPNoContent                      # 204
  #           HTTPResetContent                   # 205
  #           HTTPPartialContent                 # 206
  #       HTTPRedirection                    # 3xx
  #           HTTPMultipleChoice                 # 300
  #           HTTPMovedPermanently               # 301
  #           HTTPFound                          # 302
  #           HTTPSeeOther                       # 303
  #           HTTPNotModified                    # 304
  #           HTTPUseProxy                       # 305
  #           HTTPTemporaryRedirect              # 307
  #       HTTPClientError                    # 4xx
  #           HTTPBadRequest                     # 400
  #           HTTPUnauthorized                   # 401
  #           HTTPPaymentRequired                # 402
  #           HTTPForbidden                      # 403
  #           HTTPNotFound                       # 404
  #           HTTPMethodNotAllowed               # 405
  #           HTTPNotAcceptable                  # 406
  #           HTTPProxyAuthenticationRequired    # 407
  #           HTTPRequestTimeOut                 # 408
  #           HTTPConflict                       # 409
  #           HTTPGone                           # 410
  #           HTTPLengthRequired                 # 411
  #           HTTPPreconditionFailed             # 412
  #           HTTPRequestEntityTooLarge          # 413
  #           HTTPRequestURITooLong              # 414
  #           HTTPUnsupportedMediaType           # 415
  #           HTTPRequestedRangeNotSatisfiable   # 416
  #           HTTPExpectationFailed              # 417
  #       HTTPServerError                    # 5xx
  #           HTTPInternalServerError            # 500
  #           HTTPNotImplemented                 # 501
  #           HTTPBadGateway                     # 502
  #           HTTPServiceUnavailable             # 503
  #           HTTPGatewayTimeOut                 # 504
  #           HTTPVersionNotSupported            # 505
  # 
  # == Switching Net::HTTP versions
  # 
  # You can use net/http.rb 1.1 features (bundled with Ruby 1.6)
  # by calling HTTP.version_1_1. Calling Net::HTTP.version_1_2
  # allows you to use 1.2 features again.
  # 
  #     # example
  #     Net::HTTP.start {|http1| ...(http1 has 1.2 features)... }
  # 
  #     Net::HTTP.version_1_1
  #     Net::HTTP.start {|http2| ...(http2 has 1.1 features)... }
  # 
  #     Net::HTTP.version_1_2
  #     Net::HTTP.start {|http3| ...(http3 has 1.2 features)... }
  # 
  # This function is NOT thread-safe.
  #
  class HTTP < Protocol

    # :stopdoc:
    Revision = %q$Revision: 30571 $.split[1]
    HTTPVersion = '1.1'
    @newimpl = true
    begin
      if defined? RHO_ME
        HAVE_ZLIB=false      
      else  
        require 'zlib'
        require 'stringio'  #for our purposes (unpacking gzip) lump these together
        HAVE_ZLIB=true
      end    
    rescue LoadError
      HAVE_ZLIB=false
    end
    # :startdoc:

    # Turns on net/http 1.2 (ruby 1.8) features.
    # Defaults to ON in ruby 1.8.
    #
    # I strongly recommend to call this method always.
    #
    #   require 'net/http'
    #   Net::HTTP.version_1_2
    #
    def HTTP.version_1_2
      @newimpl = true
    end

    # Turns on net/http 1.1 (ruby 1.6) features.
    # Defaults to OFF in ruby 1.8.
    def HTTP.version_1_1
      @newimpl = false
    end

    # true if net/http is in version 1.2 mode.
    # Defaults to true.
    def HTTP.version_1_2?
      @newimpl
    end

    # true if net/http is in version 1.1 compatible mode.
    # Defaults to true.
    def HTTP.version_1_1?
      not @newimpl
    end

    class << HTTP
      alias is_version_1_1? version_1_1?   #:nodoc:
      alias is_version_1_2? version_1_2?   #:nodoc:
    end

    #
    # short cut methods
    #

    #
    # Get body from target and output it to +$stdout+.  The
    # target can either be specified as (+uri+), or as
    # (+host+, +path+, +port+ = 80); so: 
    #
    #    Net::HTTP.get_print URI.parse('http://www.example.com/index.html')
    #
    # or:
    #
    #    Net::HTTP.get_print 'www.example.com', '/index.html'
    #
    def HTTP.get_print(uri_or_host, path = nil, port = nil)
      get_response(uri_or_host, path, port) {|res|
        res.read_body do |chunk|
          $stdout.print chunk
        end
      }
      nil
    end

    # Send a GET request to the target and return the response
    # as a string.  The target can either be specified as
    # (+uri+), or as (+host+, +path+, +port+ = 80); so:
    # 
    #    print Net::HTTP.get(URI.parse('http://www.example.com/index.html'))
    #
    # or:
    #
    #    print Net::HTTP.get('www.example.com', '/index.html')
    #
    def HTTP.get(uri_or_host, path = nil, port = nil)
      get_response(uri_or_host, path, port).body
    end

    # Send a GET request to the target and return the response
    # as a Net::HTTPResponse object.  The target can either be specified as
    # (+uri+), or as (+host+, +path+, +port+ = 80); so:
    # 
    #    res = Net::HTTP.get_response(URI.parse('http://www.example.com/index.html'))
    #    print res.body
    #
    # or:
    #
    #    res = Net::HTTP.get_response('www.example.com', '/index.html')
    #    print res.body
    #
    def HTTP.get_response(uri_or_host, path = nil, port = nil, &block)
      if path
        host = uri_or_host
        new(host, port || HTTP.default_port).start {|http|
          return http.request_get(path, &block)
        }
      else
        uri = uri_or_host
        new(uri.host, uri.port).start {|http|
          return http.request_get(uri.request_uri, &block)
        }
      end
    end

    # Posts HTML form data to the +URL+.
    # Form data must be represented as a Hash of String to String, e.g:
    #
    #   { "cmd" => "search", "q" => "ruby", "max" => "50" }
    #
    # This method also does Basic Authentication iff +URL+.user exists.
    #
    # Example:
    #
    #   require 'net/http'
    #   require 'uri'
    #
    #   HTTP.post_form URI.parse('http://www.example.com/search.cgi'),
    #                  { "q" => "ruby", "max" => "50" }
    #
    def HTTP.post_form(url, params)
      req = Post.new(url.path)
      req.form_data = params
      req.basic_auth url.user, url.password if url.user
      new(url.host, url.port).start {|http|
        http.request(req)
      }
    end

    #
    # HTTP session management
    #

    # The default port to use for HTTP requests; defaults to 80.
    def HTTP.default_port
      http_default_port()
    end

    # The default port to use for HTTP requests; defaults to 80.
    def HTTP.http_default_port
      80
    end

    # The default port to use for HTTPS requests; defaults to 443.
    def HTTP.https_default_port
      443
    end

    def HTTP.socket_type   #:nodoc: obsolete
      BufferedIO
    end

    # call-seq:
    #   HTTP.start(address, port, p_addr, p_port, p_user, p_pass, &block)
    #   HTTP.start(address, port=nil, p_addr=nil, p_port=nil, p_user=nil, p_pass=nil, opt, &block)
    #
    # creates a new Net::HTTP object and opens its TCP connection and
    # HTTP session.
    #
    # Argments are following:
    # _address_ :: hostname or IP address of the server
    # _port_    :: port of the server
    # _p_addr_  :: address of proxy
    # _p_port_  :: port of proxy
    # _p_user_  :: user of proxy
    # _p_pass_  :: pass of proxy
    # _opt_     :: optional hash
    #
    # _opt_ sets following values by its accessor.
    # The keys are ca_file, ca_path, cert, cert_store, ciphers,
    # close_on_empty_response, key, open_timeout, read_timeout, ssl_timeout,
    # ssl_version, use_ssl, verify_callback, verify_depth and verify_mode.
    # If you set :use_ssl as true, you can use https and default value of
    # verify_mode is set as OpenSSL::SSL::VERIFY_PEER.
    #
    # If the optional block is given, the newly
    # created Net::HTTP object is passed to it and closed when the
    # block finishes.  In this case, the return value of this method
    # is the return value of the block.  If no block is given, the
    # return value of this method is the newly created Net::HTTP object
    # itself, and the caller is responsible for closing it upon completion.
    def HTTP.start(address, *arg, &block) # :yield: +http+
      arg.pop if opt = Hash.try_convert(arg[-1])
      port, p_addr, p_port, p_user, p_pass = *arg
      port = https_default_port if !port && opt && opt[:use_ssl]
      http = new(address, port, p_addr, p_port, p_user, p_pass)

      #if opt
      #  opt = {verify_mode: OpenSSL::SSL::VERIFY_PEER}.update(opt) if opt[:use_ssl]
      #  http.methods.grep(/\A(\w+)=\z/) do |meth|
      #    key = $1.to_sym
      #    opt.key?(key) or next
      #    http.__send__(meth, opt[key])
      #  end
     # end

      http.start(&block)
    end

    class << HTTP
      alias newobj new
    end

    # Creates a new Net::HTTP object.
    # If +proxy_addr+ is given, creates an Net::HTTP object with proxy support.
    # This method does not open the TCP connection.
    def HTTP.new(address, port = nil, p_addr = nil, p_port = nil, p_user = nil, p_pass = nil)
      h = Proxy(p_addr, p_port, p_user, p_pass).newobj(address, port)
      h.instance_eval {
        @newimpl = ::Net::HTTP.version_1_2?
      }
      h
    end

    # Creates a new Net::HTTP object for the specified +address+.
    # This method does not open the TCP connection.
    def initialize(address, port = nil)
      @address = address
      @port    = (port || HTTP.default_port)
      @curr_http_version = HTTPVersion
      @no_keepalive_server = false
      @close_on_empty_response = false
      @socket  = nil
      @started = false
      @open_timeout = nil
      @read_timeout = 60
      @debug_output = nil
      @use_ssl = false
      @ssl_context = nil
      @enable_post_connection_check = true
      @compression = nil
      @sspi_enabled = false
      if defined?(SSL_ATTRIBUTES)
        SSL_ATTRIBUTES.each do |name|
          instance_variable_set "@#{name}", nil
        end
      end
    end

    def inspect
      "#<#{self.class} #{@address}:#{@port} open=#{started?}>"
    end

    # *WARNING* This method causes serious security hole.
    # Never use this method in production code.
    #
    # Set an output stream for debugging.
    #
    #   http = Net::HTTP.new
    #   http.set_debug_output $stderr
    #   http.start { .... }
    #
    def set_debug_output(output)
      warn 'Net::HTTP#set_debug_output called after HTTP started' if started?
      @debug_output = output
    end

    # The host name to connect to.
    attr_reader :address

    # The port number to connect to.
    attr_reader :port

    # Seconds to wait until connection is opened.
    # If the HTTP object cannot open a connection in this many seconds,
    # it raises a TimeoutError exception.
    attr_accessor :open_timeout

    # Seconds to wait until reading one block (by one read(2) call).
    # If the HTTP object cannot open a connection in this many seconds,
    # it raises a TimeoutError exception.
    attr_reader :read_timeout

    # Setter for the read_timeout attribute.
    def read_timeout=(sec)
      @socket.read_timeout = sec if @socket
      @read_timeout = sec
    end

    # returns true if the HTTP session is started.
    def started?
      @started
    end

    alias active? started?   #:nodoc: obsolete

    attr_accessor :close_on_empty_response

    # returns true if use SSL/TLS with HTTP.
    def use_ssl?
      false   # redefined in net/https
    end

    # Opens TCP connection and HTTP session.
    # 
    # When this method is called with block, gives a HTTP object
    # to the block and closes the TCP connection / HTTP session
    # after the block executed.
    #
    # When called with a block, returns the return value of the
    # block; otherwise, returns self.
    #
    def start  # :yield: http
      raise IOError, 'HTTP session already opened' if @started
      if block_given?
        begin
          do_start
          return yield(self)
        ensure
          do_finish
        end
      end
      do_start
      self
    end

    def do_start
      connect
      @started = true
    end
    private :do_start

    def connect
      platform = System::get_property('platform')
      if !(use_ssl? and platform == 'Blackberry')
        D "opening connection to #{conn_address()}..."
        s = timeout(@open_timeout) { TCPSocket.open(conn_address(), conn_port()) }
        D "opened"
      end
      if use_ssl?
        ssl_parameters = Hash.new
        iv_list = instance_variables
        SSL_ATTRIBUTES.each do |name|
          ivname = "@#{name}".intern
          if iv_list.include?(ivname)
	   
            ssl_parameters[name] = value if value = instance_variable_get(ivname)
          end
        end
        if platform != 'Blackberry'
          @ssl_context = OpenSSL::SSL::SSLContext.new
          @ssl_context.set_params(ssl_parameters)
          s = OpenSSL::SSL::SSLSocket.new(s, @ssl_context)
          s.sync_close = true
        else
          s = SSLSocket.open(conn_address(), conn_port())
        end
      end
      @socket = BufferedIO.new(s)
      @socket.read_timeout = @read_timeout
      @socket.debug_output = @debug_output
      if use_ssl?
        begin
          if proxy?
            @socket.writeline sprintf('CONNECT %s:%s HTTP/%s',
                                      @address, @port, HTTPVersion)
            @socket.writeline "Host: #{@address}:#{@port}"
            if proxy_user
              credential = ["#{proxy_user}:#{proxy_pass}"].pack('m')
              credential.delete!("\r\n")
              @socket.writeline "Proxy-Authorization: Basic #{credential}"
            end
            @socket.writeline ''
            HTTPResponse.read_new(@socket).value
          end
	  if platform != 'Blackberry'
            timeout(@open_timeout) { s.connect }
            if @ssl_context.verify_mode != OpenSSL::SSL::VERIFY_NONE
              s.post_connection_check(@address)
            end
	  end   
        rescue => exception
          D "Conn close because of connect error #{exception}"
          @socket.close if @socket and not @socket.closed?
          raise exception
        end
      end
      on_connect
    end
    
    private :connect

    def on_connect
    end
    private :on_connect

{'fraction_non_alphanumeric': 0.1083503622515326, 'fraction_numerical': 0.015558238900241501, 'mean_word_length': 1.8744493392070485, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 42, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5755768', 'n_tokens_mistral': 14086, 'n_tokens_neox': 13378, 'n_words': 3616}