<commit_before>require 'rexml/document'
require 'rexml/entity'

# Fixes the rexml vulnerability disclosed at:
# http://www.ruby-lang.org/en/news/2008/08/23/dos-vulnerability-in-rexml/
# This fix is identical to rexml-expansion-fix version 1.0.1

unless REXML::VERSION > "3.1.7.2"
  module REXML
    class Entity < Child
      undef_method :unnormalized
      def unnormalized
        document.record_entity_expansion! if document
        v = value()
        return nil if v.nil?
        @unnormalized = Text::unnormalize(v, parent)
        @unnormalized
      end
    end
    class Document < Element
      @@entity_expansion_limit = 10_000
      def self.entity_expansion_limit= val
        @@entity_expansion_limit = val
      end

      def record_entity_expansion!
        @number_of_expansions ||= 0
        @number_of_expansions += 1
        if @number_of_expansions > @@entity_expansion_limit
          raise "Number of entity expansions exceeded, processing aborted."
        end
      end
    end
  end
end<commit_msg>Remove the Version check as it's not always available.<commit_after>require 'rexml/document'
require 'rexml/entity'

# Fixes the rexml vulnerability disclosed at:
# http://www.ruby-lang.org/en/news/2008/08/23/dos-vulnerability-in-rexml/
# This fix is identical to rexml-expansion-fix version 1.0.1

module REXML
  class Entity < Child
    undef_method :unnormalized
    def unnormalized
      document.record_entity_expansion! if document
      v = value()
      return nil if v.nil?
      @unnormalized = Text::unnormalize(v, parent)
      @unnormalized
    end
  end
  class Document < Element
    @@entity_expansion_limit = 10_000
    def self.entity_expansion_limit= val
      @@entity_expansion_limit = val
    end

    def record_entity_expansion!
      @number_of_expansions ||= 0
      @number_of_expansions += 1
      if @number_of_expansions > @@entity_expansion_limit
        raise "Number of entity expansions exceeded, processing aborted."
      end
    end
  end
end
{'fraction_non_alphanumeric': 0.07972097658196313, 'fraction_numerical': 0.019930244145490782, 'mean_word_length': 2.921875, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 10}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6074344', 'n_tokens_mistral': 675, 'n_tokens_neox': 626, 'n_words': 161}