<commit_before>module Baler
  module Parser
    autoload :Document, File.dirname(__FILE__) + '/parser/document'
    autoload :Collection, File.dirname(__FILE__) + '/parser/collection'
    autoload :Element, File.dirname(__FILE__) + '/parser/element'
    autoload :Adapter, File.dirname(__FILE__) + '/parser/adapter'
    autoload :Support, File.dirname(__FILE__) + '/parser/support'
    
    DOCUMENT_ADAPTERS = {:hpricot => Adapter::Hpricot::Document}
    DEFAULT_DOCUMENT_ADAPTER = DOCUMENT_ADAPTERS.keys.first

    def self.document(type_or_adapter, url, context_path = nil)
      if adapter_class = DOCUMENT_ADAPTERS[(type_or_adapter || DEFAULT_DOCUMENT_ADAPTER)]
        adapter = adapter_class.new(url)
      else
        adapter = type_or_adapter
      end

      Document.new(adapter, context_path)
    end

    def self.filter(object)
      if object.is_a? Parser::Collection
        object = object.length <= 1 ? object.first : object.to_array.map{|e| e.inner_html}
      end

      if object.is_a? Parser::Element
        object = object.inner_html
      end

      object
    end
  end
end<commit_msg>Move Baler::Parser methods into a class << self block<commit_after>module Baler
  module Parser
    autoload :Document, File.dirname(__FILE__) + '/parser/document'
    autoload :Collection, File.dirname(__FILE__) + '/parser/collection'
    autoload :Element, File.dirname(__FILE__) + '/parser/element'
    autoload :Adapter, File.dirname(__FILE__) + '/parser/adapter'
    autoload :Support, File.dirname(__FILE__) + '/parser/support'
    
    DOCUMENT_ADAPTERS = {:hpricot => Adapter::Hpricot::Document}
    DEFAULT_DOCUMENT_ADAPTER = DOCUMENT_ADAPTERS.keys.first

    class << self
      def document(type_or_adapter, url, context_path = nil)
        if adapter_class = DOCUMENT_ADAPTERS[(type_or_adapter || DEFAULT_DOCUMENT_ADAPTER)]
          adapter = adapter_class.new(url)
        else
          adapter = type_or_adapter
        end

        Document.new(adapter, context_path)
      end

      def filter(object)
        if object.is_a? Parser::Collection
          object = object.length <= 1 ? object.first : object.to_array.map{|e| e.inner_html}
        end

        if object.is_a? Parser::Element
          object = object.inner_html
        end

        object
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.10112359550561797, 'fraction_numerical': 0.000864304235090752, 'mean_word_length': 3.054290718038529, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4359154', 'n_tokens_mistral': 732, 'n_tokens_neox': 701, 'n_words': 147}