<commit_before># A minimalistic, declarative HTML scraper
class Nibbler
  attr_reader :doc
  
  # Declare a singular scraping rule
  def self.element(*args, &block)
    selector, name, delegate = parse_rule_declaration(*args, &block)
    rules[name] = [selector, delegate]
    attr_accessor name
    name
  end
  
  # Declare a plural scraping rule
  def self.elements(*args, &block)
    name = element(*args, &block)
    rules[name] << true
  end
  
  # Process data by creating a new scraper
  def self.parse(data) new(data).parse end
  
  # Initialize the parser with raw data or a document
  def initialize(data)
    @doc = self.class.convert_document(data)
    # initialize plural properties
    self.class.rules.each { |name, (s, k, plural)| send("#{name}=", []) if plural }
  end
  
  # Parse the document and save values returned by selectors
  def parse
    self.class.rules.each do |target, (selector, delegate, plural)|
      if plural
        send(target).concat @doc.search(selector).map { |i| parse_result(i, delegate) }
      else
        send("#{target}=", parse_result(@doc.at(selector), delegate))
      end
    end
    self
  end
  
  # Dump the extracted data into a hash with symbolized keys
  def to_hash
    converter = lambda { |obj| obj.respond_to?(:to_hash) ? obj.to_hash : obj }
    self.class.rules.keys.inject({}) do |hash, name|
      value = send(name)
      hash[name.to_sym] = Array === value ? value.map(&converter) : converter[value]
      hash
    end
  end
  
  protected
  
  # `delegate` is optional, but should respond to `call` or `parse`
  def parse_result(node, delegate)
    if delegate
      delegate.respond_to?(:call) ? delegate.call(node) : delegate.parse(node)
    elsif node.respond_to? :inner_text
      node.inner_text
    else
      node.to_s
    end unless node.nil?
  end
  
  private
  
  # Parsing rules declared with `element` or `elements`
  def self.rules
    @rules ||= {}
  end
  
  # Make subclasses inherit the parsing rules
  def self.inherited(subclass)
    subclass.rules.update self.rules<commit_msg>don't cast all values to strings; it's not always XML that's parsed<commit_after># A minimalistic, declarative HTML scraper
class Nibbler
  attr_reader :doc
  
  # Declare a singular scraping rule
  def self.element(*args, &block)
    selector, name, delegate = parse_rule_declaration(*args, &block)
    rules[name] = [selector, delegate]
    attr_accessor name
    name
  end
  
  # Declare a plural scraping rule
  def self.elements(*args, &block)
    name = element(*args, &block)
    rules[name] << true
  end
  
  # Process data by creating a new scraper
  def self.parse(data) new(data).parse end
  
  # Initialize the parser with raw data or a document
  def initialize(data)
    @doc = self.class.convert_document(data)
    # initialize plural properties
    self.class.rules.each { |name, (s, k, plural)| send("#{name}=", []) if plural }
  end
  
  # Parse the document and save values returned by selectors
  def parse
    self.class.rules.each do |target, (selector, delegate, plural)|
      if plural
        send(target).concat @doc.search(selector).map { |i| parse_result(i, delegate) }
      else
        send("#{target}=", parse_result(@doc.at(selector), delegate))
      end
    end
    self
  end
  
  # Dump the extracted data into a hash with symbolized keys
  def to_hash
    converter = lambda { |obj| obj.respond_to?(:to_hash) ? obj.to_hash : obj }
    self.class.rules.keys.inject({}) do |hash, name|
      value = send(name)
      hash[name.to_sym] = Array === value ? value.map(&converter) : converter[value]
      hash
    end
  end
  
  protected
  
  # `delegate` is optional, but should respond to `call` or `parse`
  def parse_result(node, delegate)
    if delegate
      delegate.respond_to?(:call) ? delegate.call(node) : delegate.parse(node)
    elsif node.respond_to? :inner_text
      node.inner_text
    else
      node
    end unless node.nil?
  end
  
  private
  
  # Parsing rules declared with `element` or `elements`
  def self.rules
    @rules ||= {}
  end
  
  # Make subclasses inherit the parsing rules
  def self.inherited(subclass)
    subclass.rules.update self.rules
{'fraction_non_alphanumeric': 0.1053763440860215, 'fraction_numerical': 0.0, 'mean_word_length': 3.0839024390243903, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4960181', 'n_tokens_mistral': 1323, 'n_tokens_neox': 1315, 'n_words': 400}