<commit_before>
# Author::    TAC (<EMAIL>)

require_relative 'yasuri_node'

module Yasuri
  class PaginateNode
    include Node

    def initialize(xpath, name, children = [], limit: nil, flatten: false)
      super(xpath, name, children)
      @flatten = flatten
      @limit = limit
    end

    def inject(agent, page, opt = {}, element = page)
      retry_count = opt[:retry_count] || Yasuri::DefaultRetryCount

      raise NotImplementedError.new("PagenateNode inside StructNode, Not Supported") if page != element

      child_results = []
      limit = @limit.nil? ? Float::MAX : @limit
      while page
        child_results_kv = @children.map do |child_node|
          child_name = Yasuri.NodeName(child_node.name, opt)
          [child_name, child_node.inject(agent, page, opt)]
        end
        child_results << Hash[child_results_kv]

        link = page.search(@xpath).first
        break if link == nil

        link_button = Mechanize::Page::Link.new(link, agent, page)
        page = Yasuri.with_retry(retry_count) { link_button.click }
        break if (limit -= 1) <= 0
      end

      if @flatten == true
        return child_results.map{|h| h.values}.flatten
      end

      child_results<commit_msg>Add todo comment<commit_after>
# Author::    TAC (<EMAIL>)

require_relative 'yasuri_node'

module Yasuri
  class PaginateNode
    include Node

    def initialize(xpath, name, children = [], limit: nil, flatten: false)
      super(xpath, name, children)
      @flatten = flatten
      @limit = limit
    end

    def inject(agent, page, opt = {}, element = page)
      retry_count = opt[:retry_count] || Yasuri::DefaultRetryCount

      raise NotImplementedError.new("PagenateNode inside StructNode, Not Supported") if page != element

      child_results = []
      limit = @limit.nil? ? Float::MAX : @limit
      while page
        child_results_kv = @children.map do |child_node|
          child_name = Yasuri.NodeName(child_node.name, opt)
          [child_name, child_node.inject(agent, page, opt)]
        end
        child_results << Hash[child_results_kv]

        link = page.search(@xpath).first # Todo raise:  link is not found
        break if link == nil

        link_button = Mechanize::Page::Link.new(link, agent, page)
        page = Yasuri.with_retry(retry_count) { link_button.click }
        break if (limit -= 1) <= 0
      end

      if @flatten == true
        return child_results.map{|h| h.values}.flatten
      end

      child_results
{'fraction_non_alphanumeric': 0.10691318327974277, 'fraction_numerical': 0.001607717041800643, 'mean_word_length': 2.698365527488856, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1346095', 'n_tokens_mistral': 781, 'n_tokens_neox': 756, 'n_words': 195}