<commit_before>require 'watir-webdriver'

class Razor
  attr_reader :webdriver
  def initialize(options={})
    @webdriver = Watir::Browser.new(options[:blade] || :firefox)
    ObjectSpace.define_finalizer( self, self.class.finalize(@webdriver) )
  end
  def goto(url)
    @webdriver.goto(url)
  end
  def url
    @webdriver.url
  end
  def enter_text(xpath, text)
    @webdriver.text_field(:xpath, xpath).set text
  end
  def submit
    @webdriver.form(:xpath, "//form").submit
  end
  def element(xpath)
    @webdriver.element_by_xpath(xpath)
  end
  def input(xpath)
    @webdriver.input(:xpath, xpath)
  end
  def click(xpath)
    @webdriver.element_by_xpath(xpath).click
  end
  def shave(options={}, &block)
    Shave.new(@webdriver, options, &block).evaluate
  end
  def close
    @webdriver.close
  end

  def self.finalize(webdriver)
    proc { webdriver.close }
  end
end

class Shave
  def initialize(webdriver, options, &block)
    @webdriver = webdriver
    @options = options
    @options[:number_of_steps]||=10
    @options[:step_time]||=0.5
    @options[:validation_limit]||=10
    @arrays = []
    @values = []
    self.instance_eval &block
  end

  def validate(&block)
    @validator = block
  end

  def next_page(xpath)
    @next_page_xpath = xpath
  end

  def value(name, xpath, &block)
    @values << [name,xpath,block]
  end

  def array(name, xpath, &block)
    @arrays << [name,xpath,block]
  end

  def evaluate
    result = {}
    validation_attempts = 0
    while(true) do
      #page scrape
      page_results = evaluate_values.merge(evaluate_arrays)
      new_result = merge_values(result, page_results)

      #next page
      next_page = nil
      unless @next_page_xpath == nil
        next_page = @webdriver.element_by_xpath(@next_page_xpath)

        break unless next_page.exists?
      end

      # validation
      if @validator != nil
        unless @validator.call(page_results)
          validation_attempts += 1
          raise "Validation limit of #{@options[:validation_limit]} reached for url #{@webdriver.url}" if(validation_attempts >= @options[:validation_limit])
          sleep @options[:step_time]
          next
        end
      end
      validation_attempts = 0

      result = new_result

      # goto next page
      break if @next_page_xpath == nil
      next_page.click
    end
    result
  end


private


  # Two hashs of {:test => [1]} {:test => [2,3]}
  # become: {:test => [1,2,3]}
  def merge_values(f, s)<commit_msg>Added parameters to Watir::Browser.new().  Added an ipchicken example.<commit_after>require 'watir-webdriver'

class Razor
  attr_reader :webdriver
  def initialize(options={})
    @webdriver = Watir::Browser.new(options.delete(:blade) || :firefox, options)
    ObjectSpace.define_finalizer( self, self.class.finalize(@webdriver) )
  end
  def goto(url)
    @webdriver.goto(url)
  end
  def url
    @webdriver.url
  end
  def enter_text(xpath, text)
    @webdriver.text_field(:xpath, xpath).set text
  end
  def submit
    @webdriver.form(:xpath, "//form").submit
  end
  def element(xpath)
    @webdriver.element_by_xpath(xpath)
  end
  def input(xpath)
    @webdriver.input(:xpath, xpath)
  end
  def click(xpath)
    @webdriver.element_by_xpath(xpath).click
  end
  def shave(options={}, &block)
    Shave.new(@webdriver, options, &block).evaluate
  end
  def close
    @webdriver.close
  end

  def self.finalize(webdriver)
    proc { webdriver.close }
  end
end

class Shave
  def initialize(webdriver, options, &block)
    @webdriver = webdriver
    @options = options
    @options[:number_of_steps]||=10
    @options[:step_time]||=0.5
    @options[:validation_limit]||=10
    @arrays = []
    @values = []
    self.instance_eval &block
  end

  def validate(&block)
    @validator = block
  end

  def next_page(xpath)
    @next_page_xpath = xpath
  end

  def value(name, xpath, &block)
    @values << [name,xpath,block]
  end

  def array(name, xpath, &block)
    @arrays << [name,xpath,block]
  end

  def evaluate
    result = {}
    validation_attempts = 0
    while(true) do
      #page scrape
      page_results = evaluate_values.merge(evaluate_arrays)
      new_result = merge_values(result, page_results)

      #next page
      next_page = nil
      unless @next_page_xpath == nil
        next_page = @webdriver.element_by_xpath(@next_page_xpath)

        unless next_page.exists?
          result = new_result
          break
        end
      end

      # validation
      if @validator != nil
        unless @validator.call(page_results)
          validation_attempts += 1
          raise "Validation limit of #{@options[:validation_limit]} reached for url #{@webdriver.url}" if(validation_attempts >= @options[:validation_limit])
          sleep @options[:step_time]
          next
        end
      end
      validation_attempts = 0

      result = new_result

      # goto next page
      break if @next_page_xpath == nil
      next_page.click
    end
    result
  end


private


  # Two hashs of {:test => [1]} {:test => [2,3]}
  # become: {:test => [1,2,3]}
  def merge_values(f, s)
{'fraction_non_alphanumeric': 0.11274798664309565, 'fraction_numerical': 0.0058927519151443725, 'mean_word_length': 2.9411764705882355, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1717869', 'n_tokens_mistral': 1764, 'n_tokens_neox': 1701, 'n_words': 384}