<commit_before>require 'net/http'
require 'nokogiri'

module Otomo
  class Robot

    attr_accessor :scheme, :host, :port

    attr_accessor :cookies, :referer

    attr_reader :http

    def raw_mode=raw_mode; @raw_mode=raw_mode; end
    def raw_mode?; @raw_mode; end

    attr_accessor :header

    def initialize path, header={}
      @header = header
      @raw_mode = false

      if path.is_a?(String)
        uri = URI(path)
        @scheme = uri.scheme
        @host = uri.host
        @port = uri.port
      end
    end

    def use_ssl= x; @use_ssl = x; end

    def use_ssl?
      if @use_ssl.nil?
        (@scheme == "https")
      else
        @use_ssl
      end
    end

    def connect
      @http = Net::HTTP.new(@host, port)
      @http.use_ssl = (@scheme == "https")
    end

    def full_path path
      if path =~ URI::regexp
        path
      else
        "#{@scheme}://#{@host}" + File.join("/", path)
      end
    end

    def html_document string
      Nokogiri::HTML(string)
    end

    def put path, data={}
      request "PUT", path, data
    end

    def patch path, data={}
      request "PATCH", path, data
    end

    def get path, data={}
      path, query = get_query_path path
      request "GET", path + query_encoded_data(data, query),nil
    end

    def delete path, data={}
      path, query = get_query_path path
      request "PATH", path + query_encoded_data(data, query), nil
    end

    def request method, path, data={}
      resp = http.send_request(method, path, data, prepare_headers)

      handle_response resp do
        set_cookies resp.get_fields('set-cookie')
        @referer = full_path(path)
      end
    end


    def post path, data={}
      if data.is_a?(Hash)
        data = Otomo.hash_to_www_url_encoded(data)
      end

      resp = http.post(File.join("/", path), data, prepare_headers)

      File.write("tmp.html", resp.body)

      handle_response resp do
        set_cookies resp.get_fields('set-cookie')
        @referer = full_path(path)
      end
    end

    def add_cookie key, value
      @cookies ||= {}
      @cookies[key] = value
    end

    def remove_cookie c
      @cookies.delete(c)
    end

    def clear_cookies
      @cookies = {}
    end

private
<commit_msg>Remove a debug line I forgot<commit_after>require 'net/http'
require 'nokogiri'

module Otomo
  class Robot

    attr_accessor :scheme, :host, :port

    attr_accessor :cookies, :referer

    attr_reader :http

    def raw_mode=raw_mode; @raw_mode=raw_mode; end
    def raw_mode?; @raw_mode; end

    attr_accessor :header

    def initialize path, header={}
      @header = header
      @raw_mode = false

      if path.is_a?(String)
        uri = URI(path)
        @scheme = uri.scheme
        @host = uri.host
        @port = uri.port
      end
    end

    def use_ssl= x; @use_ssl = x; end

    def use_ssl?
      if @use_ssl.nil?
        (@scheme == "https")
      else
        @use_ssl
      end
    end

    def connect
      @http = Net::HTTP.new(@host, port)
      @http.use_ssl = (@scheme == "https")
    end

    def full_path path
      if path =~ URI::regexp
        path
      else
        "#{@scheme}://#{@host}" + File.join("/", path)
      end
    end

    def html_document string
      Nokogiri::HTML(string)
    end

    def put path, data={}
      request "PUT", path, data
    end

    def patch path, data={}
      request "PATCH", path, data
    end

    def get path, data={}
      path, query = get_query_path path
      request "GET", path + query_encoded_data(data, query),nil
    end

    def delete path, data={}
      path, query = get_query_path path
      request "PATH", path + query_encoded_data(data, query), nil
    end

    def request method, path, data={}
      resp = http.send_request(method, path, data, prepare_headers)

      handle_response resp do
        set_cookies resp.get_fields('set-cookie')
        @referer = full_path(path)
      end
    end


    def post path, data={}
      if data.is_a?(Hash)
        data = Otomo.hash_to_www_url_encoded(data)
      end

      resp = http.post(File.join("/", path), data, prepare_headers)

      handle_response resp do
        set_cookies resp.get_fields('set-cookie')
        @referer = full_path(path)
      end
    end

    def add_cookie key, value
      @cookies ||= {}
      @cookies[key] = value
    end

    def remove_cookie c
      @cookies.delete(c)
    end

    def clear_cookies
      @cookies = {}
    end

private

{'fraction_non_alphanumeric': 0.10723860589812333, 'fraction_numerical': 0.0, 'mean_word_length': 2.1307692307692307, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4288904', 'n_tokens_mistral': 1511, 'n_tokens_neox': 1475, 'n_words': 401}