<commit_before>require 'nori'

module XMLMunger

  class Parser
    attr_accessor :xml

    def initialize xml, nori_options = {}
      unless xml.is_a?(Hash)
        unless xml.is_a?(String)
          raise ArgumentError.new("Argument xml should be a Hash or String (XML file).")
        end
        @xml = ::Nori.new(NoriConstants.default_options.merge(nori_options)).parse(xml)
      else
        @xml = xml
      end
    end

    def run options = {}
      raise TypeError.new("options argument must be a hash") unless options.is_a?(Hash)
      options = default_options.merge options

      @xml = options[:filter].inject(xml) { |hash, key| hash[key] }
      array_of_arrays_of_data = NestedHash[xml].map_values_with_route do |route, value|
        next if route.any? { |r| r =~ /@/ } && !options[:attributes] # skip attributes
        route.map! { |s| s.to_s.tr(options[:strip_chars], '') }
        route = route.join(options[:sep])
        key = options[:prefix] + route
        # TODO: Should we parse out attributes from nested tags?

        [key, value]
      end.compact.reject { |k, v| options[:prohibited_types].any? { |type| v.is_a?(type) } } 
      Hash[array_of_arrays_of_data]
    end

    protected

    def default_options
      {
        prefix: '',
        filter: [],
        sep: '_',
        strip_chars: '',  # Whether to strip any characters from route names
        attributes: true, # Whether or not to parse XML tag attributes
        prohibited_types: [Array]<commit_msg>don't destructively change xml when filtering<commit_after>require 'nori'

module XMLMunger

  class Parser
    attr_accessor :xml

    def initialize xml, nori_options = {}
      unless xml.is_a?(Hash)
        unless xml.is_a?(String)
          raise ArgumentError.new("Argument xml should be a Hash or String (XML file).")
        end
        @xml = ::Nori.new(NoriConstants.default_options.merge(nori_options)).parse(xml)
      else
        @xml = xml
      end
    end

    def run options = {}
      raise TypeError.new("options argument must be a hash") unless options.is_a?(Hash)
      options = default_options.merge options

      filtered = options[:filter].inject(xml) { |hash, key| hash[key] }
      array_of_arrays_of_data = NestedHash[filtered].map_values_with_route do |route, value|
        next if route.any? { |r| r =~ /@/ } && !options[:attributes] # skip attributes
        route.map! { |s| s.to_s.tr(options[:strip_chars], '') }
        route = route.join(options[:sep])
        key = options[:prefix] + route
        # TODO: Should we parse out attributes from nested tags?

        [key, value]
      end.compact.reject { |k, v| options[:prohibited_types].any? { |type| v.is_a?(type) } }
      Hash[array_of_arrays_of_data]
    end

    protected

    def default_options
      {
        prefix: '',
        filter: [],
        sep: '_',
        strip_chars: '',  # Whether to strip any characters from route names
        attributes: true, # Whether or not to parse XML tag attributes
        prohibited_types: [Array]
{'fraction_non_alphanumeric': 0.11563731931668857, 'fraction_numerical': 0.0, 'mean_word_length': 2.6206896551724137, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 23}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1592517', 'n_tokens_mistral': 936, 'n_tokens_neox': 919, 'n_words': 248}