<commit_before>require "nokogiri"
require "daimon_skycrawlers/crawler"

module DaimonSkycrawlers
  module Processor
    #
    # Web spider class.
    # By default extract all links and follow.
    #
    # @example Google search result (2016-11-29)
    #   spider = DaimonSkycrawlers::Processor::Spider.new
    #   spider.configure do |s|
    #     s.link_rules = ".g .r a"
    #     s.extract_link do |element|
    #       element["data-href"]
    #     end
    #     s.link_message = { next: "detail" }
    #     s.next_page_link_rules = "a#pnnext"
    #     s.next_page_link_message = { next: "spider" }
    #   end
    #
    class Spider < Base
      # @!attribute [rw] enqueue
      #   If true enqueue found links
      #
      # @!attribute [rw] link_rules
      #   same as Nokogiri::XML::DocumentFragment#search
      #   In generally, we can set XPath or CSS selector.
      #
      # @!attribute [rw] next_page_link_rules
      #   same as Nokogiri::XML::DocumentFragment#search
      #   In generally, we can set XPath or CSS selector.
      #
      attr_accessor :enqueue, :link_rules, :next_page_link_rules

      # @!attribute [w] link_message
      #   Specify hash literal to propagate arbitrary data next crawler/processor.
      #   This is for filtering message before crawler/processor processes the message.
      #
      # @!attribute [w] next_page_link_message
      #   Specify hash literal to propagate arbitrary data next crawler/processor.
      #   This is for filtering message before crawler/processor processes the message.
      attr_writer :link_message, :next_page_link_message

      def initialize
        super
        @link_filters = []
        @doc = nil
        @links = nil
        @enqueue = true
        @link_rules = ["a"]
        @extract_link = ->(element) { element["href"] }
        @link_message = {}
        @next_page_link_rules = nil
        @extract_next_page_link = ->(element) { element["href"] }
        @next_page_link_message = {}
      end

      #
      # Append filter to reduce links found by link_rules
      #
      # @param filter [Object] Filter object that has call method
      # @yield [message] Similar to Array#select
      # @yieldparam message [Hash]
      #
      def append_link_filter(filter = nil, &block)
        if block_given?
          @link_filters << block
        else
          @link_filters << filter if filter.respond_to?(:call)
        end
      end

      #
      # Register block to process element found by DaimonSkycrawlers::Processor::Spider#link_rules
      #
      # @yield [element]
      # @yieldparam element [Object]
      # @example Default
      #   ->(element) { element["href"] }
      #
      def extract_link(&block)
        @extract_link = block
      end

      #
      # Register block to process element found by DaimonSkycrawlers::Processor::Spider#next_page_link_rules
      #
      # @yield [element]
      # @yieldparam element [Object]
      # @example Default
      #   ->(element) { element["href"] }
      #
      def extract_next_page_link(&block)
        @extract_next_page_link = block
      end

      #
      # @param message [Hash] Must have key :url, :depth
      #
      def call(message)
        key_url = message[:url]
        depth = Integer(message[:depth] || 2)
        return if depth <= 1
        page = storage.read(key_url, message)
        @doc = Nokogiri::HTML(page.body)
        new_message = {
          depth: depth - 1,
        }
        link_message = new_message.merge(@link_message)
        links.each do |url|
          enqueue_url(url, link_message)
        end
        next_page_url = next_page_link
        if next_page_url
          next_page_link_message = new_message.merge(@next_page_link_message)
          enqueue_url(next_page_url, next_page_link_message)
        end<commit_msg>spider: Do nothing if read nothing from storage<commit_after>require "nokogiri"
require "daimon_skycrawlers/crawler"

module DaimonSkycrawlers
  module Processor
    #
    # Web spider class.
    # By default extract all links and follow.
    #
    # @example Google search result (2016-11-29)
    #   spider = DaimonSkycrawlers::Processor::Spider.new
    #   spider.configure do |s|
    #     s.link_rules = ".g .r a"
    #     s.extract_link do |element|
    #       element["data-href"]
    #     end
    #     s.link_message = { next: "detail" }
    #     s.next_page_link_rules = "a#pnnext"
    #     s.next_page_link_message = { next: "spider" }
    #   end
    #
    class Spider < Base
      # @!attribute [rw] enqueue
      #   If true enqueue found links
      #
      # @!attribute [rw] link_rules
      #   same as Nokogiri::XML::DocumentFragment#search
      #   In generally, we can set XPath or CSS selector.
      #
      # @!attribute [rw] next_page_link_rules
      #   same as Nokogiri::XML::DocumentFragment#search
      #   In generally, we can set XPath or CSS selector.
      #
      attr_accessor :enqueue, :link_rules, :next_page_link_rules

      # @!attribute [w] link_message
      #   Specify hash literal to propagate arbitrary data next crawler/processor.
      #   This is for filtering message before crawler/processor processes the message.
      #
      # @!attribute [w] next_page_link_message
      #   Specify hash literal to propagate arbitrary data next crawler/processor.
      #   This is for filtering message before crawler/processor processes the message.
      attr_writer :link_message, :next_page_link_message

      def initialize
        super
        @link_filters = []
        @doc = nil
        @links = nil
        @enqueue = true
        @link_rules = ["a"]
        @extract_link = ->(element) { element["href"] }
        @link_message = {}
        @next_page_link_rules = nil
        @extract_next_page_link = ->(element) { element["href"] }
        @next_page_link_message = {}
      end

      #
      # Append filter to reduce links found by link_rules
      #
      # @param filter [Object] Filter object that has call method
      # @yield [message] Similar to Array#select
      # @yieldparam message [Hash]
      #
      def append_link_filter(filter = nil, &block)
        if block_given?
          @link_filters << block
        else
          @link_filters << filter if filter.respond_to?(:call)
        end
      end

      #
      # Register block to process element found by DaimonSkycrawlers::Processor::Spider#link_rules
      #
      # @yield [element]
      # @yieldparam element [Object]
      # @example Default
      #   ->(element) { element["href"] }
      #
      def extract_link(&block)
        @extract_link = block
      end

      #
      # Register block to process element found by DaimonSkycrawlers::Processor::Spider#next_page_link_rules
      #
      # @yield [element]
      # @yieldparam element [Object]
      # @example Default
      #   ->(element) { element["href"] }
      #
      def extract_next_page_link(&block)
        @extract_next_page_link = block
      end

      #
      # @param message [Hash] Must have key :url, :depth
      #
      def call(message)
        key_url = message[:url]
        depth = Integer(message[:depth] || 2)
        return if depth <= 1
        page = storage.read(key_url, message)
        unless page
          log.warn("Could not read page: url=#{message[:url]}, key=#{message[:key]}")
          return
        end
        @doc = Nokogiri::HTML(page.body)
        new_message = {
          depth: depth - 1,
        }
        link_message = new_message.merge(@link_message)
        links.each do |url|
          enqueue_url(url, link_message)
        end
        next_page_url = next_page_link
        if next_page_url
          next_page_link_message = new_message.merge(@next_page_link_message)
          enqueue_url(next_page_url, next_page_link_message)
        end
{'fraction_non_alphanumeric': 0.09444798978940651, 'fraction_numerical': 0.0028079132099553287, 'mean_word_length': 2.2154288059089042, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4321201', 'n_tokens_mistral': 2417, 'n_tokens_neox': 2356, 'n_words': 596}