lib/spidr/agent/events.rb
def every_html_doc
      every_page do |page|
        if (block_given? && page.html?)
          if (doc = page.doc)
            yield doc
          end
        end
      end
    end
def every_html_doc
      every_page do |page|
        if (block_given? && page.html?)
          if (doc = page.doc)
            yield doc
          end
        end
      end
    end
Pass every HTML document that the agent parses to a given block.

 @yield [doc]
   The block will be passed every HTML document parsed.

 @yieldparam [Nokogiri::HTML::Document] doc
   A parsed HTML document.

 @see http://nokogiri.rubyforge.org/nokogiri/Nokogiri/HTML/Document.html