<commit_before>require 'htmlentities'
require 'ox'
require 'stringio'

module Badgerfish
  class OxSaxParser < Ox::Sax

    def load(xml, options = {})
      @decode_html_entities =
      if options[:decode_html_entities].nil?
        true
      else
        !!options[:decode_html_entities]
      end

      @html_entities_coder = HTMLEntities.new
      @remove_namespaces = !!options[:remove_namespaces]
      @result = @root = {}
      @parents = []
      Ox.sax_parse(self, StringIO.new(xml))
      @result
    end

    #
    # sax callbacks
    #
    def start_element(name)
      new_element = {}
      name = name.to_s # force string representation of symbols

      if @root[name].nil?
        @root[name] = new_element
      else
        @root[name] = [@root[name]] unless @root[name].is_a?(Array)
        @root[name] << new_element
      end

      @parents << @root
      @root = new_element
    end

    def end_element(name)
      (@root = @parents.pop).inject(@root) do |hash, (key, value)|
        hash[key] = nil if value.empty?
        hash
      end
    end

    def attr(name, value)
      if @remove_namespaces || !name.to_s.start_with?('xmlns')
        @root["@#{name}"] = value
      else
        @root['@xmlns'] ||= {}

        if name.to_s.start_with? 'xmlns:'
          @root['@xmlns'][name[6, name.length]] = value
        else
          @root['@xmlns']['$'] = value
        end
      end
    end

    def text(value)
      @root['$'] = @decode_html_entities ? @html_entities_coder.decode(value) : value
    end
  end
end<commit_msg>Added support for empty/self closing tags<commit_after>require 'htmlentities'
require 'ox'
require 'stringio'

module Badgerfish
  class OxSaxParser < Ox::Sax

    def load(xml, options = {})
      @decode_html_entities =
      if options[:decode_html_entities].nil?
        true
      else
        !!options[:decode_html_entities]
      end

      @html_entities_coder = HTMLEntities.new
      @remove_namespaces = !!options[:remove_namespaces]
      @result = @root = {}
      @parents = []
      Ox.sax_parse(self, StringIO.new(xml))
      @result
    end

    #
    # sax callbacks
    #
    def start_element(name)
      new_element = {}
      name = name.to_s # force string representation of symbols

      if @root[name].nil?
        @root[name] = new_element
      else
        @root[name] = [@root[name]] unless @root[name].is_a?(Array)
        @root[name] << new_element
      end

      @parents << @root
      @root = new_element
    end

    def end_element(name)
      (@root = @parents.pop).inject(@root) do |hash, (key, value)|
        hash[key] = nil if value.nil? || value.empty?
        hash
      end
    end

    def attr(name, value)
      if @remove_namespaces || !name.to_s.start_with?('xmlns')
        @root["@#{name}"] = value
      else
        @root['@xmlns'] ||= {}

        if name.to_s.start_with? 'xmlns:'
          @root['@xmlns'][name[6, name.length]] = value
        else
          @root['@xmlns']['$'] = value
        end
      end
    end

    def text(value)
      @root['$'] = @decode_html_entities ? @html_entities_coder.decode(value) : value
    end
  end
end
{'fraction_non_alphanumeric': 0.13449367088607594, 'fraction_numerical': 0.0006329113924050633, 'mean_word_length': 2.285862785862786, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 14}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2466871', 'n_tokens_mistral': 1053, 'n_tokens_neox': 1054, 'n_words': 219}