<commit_before>require 'XMLStreamObjectParser'
require 'XMLSubElements'

module XMLCodec
  # This class should be inherited from to create classes that are able to
  # import and export XML elements and their children. It provides three main
  # functions: xmlattr, xmlsubel and xmlsubel_mult.
  #
  # To create an importer/exporter for a XML format all that's needed is to 
  # create a class for each of the elements and then declare their atributes and
  # subelements.
  #
  # Two other functions have an important role. elname declares the name of the
  # XML element the class represents. elwithvalue declares that the element
  # has no subelements and includes only text content.
  #
  # After the class is defined import_xml can be used to import the content from
  # a REXML Element or Document and create_xml can be used to create the XML DOM
  # of the element as a child to a REXML Element or Document. For big documents
  # these are usually too slow and memory hungry, using xml_text to export to 
  # XML and import_xml_text to import XML are probably better ideas. 
  # import_xml_text is just a utility function around XMLStreamObjectParser,
  # that allow more flexible stream parsing of XML files while still using the
  # same XMLElement objects.
  #
  # <b>WARNING</b>: This API is still very much a work in progress and very
  # rough in certain places. Changes will surely be made.
  class XMLElement
    INDENT_STR = '  '
    CACHE = {}
  
    attr_accessor :element_id, :parent_id, :__xml_text
    attr_accessor :__parent
    ## A xmlsubel is any subelement of a given element

  private
    # Class level variable to hold the list of subelements
    def self.xmlsubels
      @xmlsubels ||=[]
    end
    
    # Class level variable to hold the list of subelements that are multiple
    def self.xmlsubelmultiples
      @xmlsubelmultiples ||=[]
    end
    
    # Class level variable that holds the list of attributes
    def self.xmlattrs
      @xmlattrs ||=[]
    end
  
    # Add a name as being a subelement (mult or single)
    def self._xmlsubel(name)
      self.xmlsubels << name
    end
    
    # Add a xmlsubel type attribute
    def self.xmlsubel(name) #:doc:
      name = name.to_sym
      self._xmlsubel(name)
      attr_reader name
      define_method((name.to_s+"=").to_sym) { |value|
        if value.is_a? String or value.is_a? Fixnum
          elclass = XMLCodec::XMLElement.get_element_class(name)
          value = elclass.new(value)
        end
        value.__parent = self if value
        instance_variable_set "@#{name}", value
      }
    end
    
    # Add a xmlsubel_mult type attribute (wrapper around attr_accessor)
    def self.xmlsubel_mult(name) #:doc:
      self._xmlsubel(name)
      self.xmlsubelmultiples << name
      define_method(name){
        if not self.instance_variables.index("@#{name}")
          instance_variable_set "@#{name}", XMLSubElements.new(self)
        end
        instance_variable_get "@#{name}"
      }
    end
    
    # Iterates over the object's XML subelements
    def self.each_subel
      if not self.instance_variables.index("@__subel_names")
        @__subel_names = []
        # Iterate all the superclasses that are still children of XMLElement
        # and iterate each of the subelements
        c = self
        while c.ancestors.index(XMLCodec::XMLElement)
          c.xmlsubels.each {|name| @__subel_names << name}
          c = c.superclass
        end
      end
      @__subel_names.each {|name| yield name}
    end

    # Iterate all the superclasses that are still children of XMLElement
    # and check if any of them have the subelement mult defined
    def self.subel_mult?(element)
      if not self.instance_variables.index("@__subel_mult_names")
        names = []
        c = self
        while c.ancestors.index(XMLCodec::XMLElement)
          c.xmlsubelmultiples.each {|name| names << name}
          c = c.superclass
        end
        @__subel_mult_names = names
      end
      return @__subel_mult_names.index(element)? true : false
    end
    
    # Iterate all the superclasses that are still children of XMLElement
    # and check if any of them have any subelements handled by this class
    def get_subel(elclass)
      names = elclass.get_elnames
      c = self.class
      while c.ancestors.index(XMLCodec::XMLElement)
        names.each do |name|
          if c.xmlsubels.index(name.to_sym)
            return names[0].to_sym
          end
        end
        c = c.superclass
      end
      return nil
    end
  
    # Iterates over the object's XML atributes
    def self.each_attr    
      if not self.instance_variables.index("@__attr_names")
        names = []
        # Iterate all the superclasses that are still children of EADElement
        # and iterate each of the attributes
        c = self
        while c.ancestors.index(XMLCodec::XMLElement)
          c.xmlattrs.each {|name| names << name}
          c = c.superclass
        end
        @__attr_names = names
      end
      
      @__attr_names.each {|name| yield name}
    end<commit_msg>Don't use #each to join arrays.<commit_after>require 'XMLStreamObjectParser'
require 'XMLSubElements'

module XMLCodec
  # This class should be inherited from to create classes that are able to
  # import and export XML elements and their children. It provides three main
  # functions: xmlattr, xmlsubel and xmlsubel_mult.
  #
  # To create an importer/exporter for a XML format all that's needed is to 
  # create a class for each of the elements and then declare their atributes and
  # subelements.
  #
  # Two other functions have an important role. elname declares the name of the
  # XML element the class represents. elwithvalue declares that the element
  # has no subelements and includes only text content.
  #
  # After the class is defined import_xml can be used to import the content from
  # a REXML Element or Document and create_xml can be used to create the XML DOM
  # of the element as a child to a REXML Element or Document. For big documents
  # these are usually too slow and memory hungry, using xml_text to export to 
  # XML and import_xml_text to import XML are probably better ideas. 
  # import_xml_text is just a utility function around XMLStreamObjectParser,
  # that allow more flexible stream parsing of XML files while still using the
  # same XMLElement objects.
  #
  # <b>WARNING</b>: This API is still very much a work in progress and very
  # rough in certain places. Changes will surely be made.
  class XMLElement
    INDENT_STR = '  '
    CACHE = {}
  
    attr_accessor :element_id, :parent_id, :__xml_text
    attr_accessor :__parent
    ## A xmlsubel is any subelement of a given element

  private
    # Class level variable to hold the list of subelements
    def self.xmlsubels
      @xmlsubels ||=[]
    end
    
    # Class level variable to hold the list of subelements that are multiple
    def self.xmlsubelmultiples
      @xmlsubelmultiples ||=[]
    end
    
    # Class level variable that holds the list of attributes
    def self.xmlattrs
      @xmlattrs ||=[]
    end
  
    # Add a name as being a subelement (mult or single)
    def self._xmlsubel(name)
      self.xmlsubels << name
    end
    
    # Add a xmlsubel type attribute
    def self.xmlsubel(name) #:doc:
      name = name.to_sym
      self._xmlsubel(name)
      attr_reader name
      define_method((name.to_s+"=").to_sym) { |value|
        if value.is_a? String or value.is_a? Fixnum
          elclass = XMLCodec::XMLElement.get_element_class(name)
          value = elclass.new(value)
        end
        value.__parent = self if value
        instance_variable_set "@#{name}", value
      }
    end
    
    # Add a xmlsubel_mult type attribute (wrapper around attr_accessor)
    def self.xmlsubel_mult(name) #:doc:
      self._xmlsubel(name)
      self.xmlsubelmultiples << name
      define_method(name){
        if not self.instance_variables.index("@#{name}")
          instance_variable_set "@#{name}", XMLSubElements.new(self)
        end
        instance_variable_get "@#{name}"
      }
    end
    
    # Iterates over the object's XML subelements
    def self.each_subel
      if not self.instance_variables.index("@__subel_names")
        names = []
        # Iterate all the superclasses that are still children of XMLElement
        # and iterate each of the subelements
        c = self
        while c.ancestors.index(XMLCodec::XMLElement)
          names += c.xmlsubels
          c = c.superclass
        end
        @__subel_names = names
      end
      @__subel_names.each {|name| yield name}
    end

    # Iterate all the superclasses that are still children of XMLElement
    # and check if any of them have the subelement mult defined
    def self.subel_mult?(element)
      if not self.instance_variables.index("@__subel_mult_names")
        names = []
        c = self
        while c.ancestors.index(XMLCodec::XMLElement)
          names += c.xmlsubelmultiples
          c = c.superclass
        end
        @__subel_mult_names = names
      end
      return @__subel_mult_names.index(element)? true : false
    end
    
    # Iterate all the superclasses that are still children of XMLElement
    # and check if any of them have any subelements handled by this class
    def get_subel(elclass)
      names = elclass.get_elnames
      c = self.class
      while c.ancestors.index(XMLCodec::XMLElement)
        names.each do |name|
          if c.xmlsubels.index(name.to_sym)
            return names[0].to_sym
          end
        end
        c = c.superclass
      end
      return nil
    end
  
    # Iterates over the object's XML atributes
    def self.each_attr    
      if not self.instance_variables.index("@__attr_names")
        names = []
        # Iterate all the superclasses that are still children of EADElement
        # and iterate each of the attributes
        c = self
        while c.ancestors.index(XMLCodec::XMLElement)
          names += c.xmlattrs
          c = c.superclass
        end
        @__attr_names = names
      end
      
      @__attr_names.each {|name| yield name}
    end
{'fraction_non_alphanumeric': 0.06616778259157148, 'fraction_numerical': 0.00019692792437967703, 'mean_word_length': 2.518184967093869, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 58}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6297569', 'n_tokens_mistral': 2961, 'n_tokens_neox': 2961, 'n_words': 1056}