<commit_before>module LLT
  module TeiHandler
    class PreProcessor
      require 'nokogiri'

      def initialize(document)
        @document = Nokogiri::XML(document)
        try_to_find_tei_root unless is_tei?
      end

      def to_xml
        without_duplicate_declaration(@document.to_xml).strip
      end

      def is_tei?
        @document.root.name =~ /^TEI/
      end

      def ignore_nodes(*nodes)
        @document.search(*nodes).each(&:remove)
        @document
      end

      alias_method :remove_nodes, :ignore_nodes

      private

      DUPLICATE_XML_VERSION = /<\?xml version=.*\?>\s*{2}/m
      def without_duplicate_declaration(doc)
        if doc.match(DUPLICATE_XML_VERSION)
          doc.sub(/^<\?xml version=.*\?>/, '')
        else
          doc
        end
      end

      def try_to_find_tei_root
        tei = @document.xpath('//*[name() = "TEI" or name() = "TEI.2"]').first
        if tei
          if RUBY_ENGINE == "jruby"<commit_msg>Explain the jruby divergence<commit_after>module LLT
  module TeiHandler
    class PreProcessor
      require 'nokogiri'

      def initialize(document)
        @document = Nokogiri::XML(document)
        try_to_find_tei_root unless is_tei?
      end

      def to_xml
        without_duplicate_declaration(@document.to_xml).strip
      end

      def is_tei?
        @document.root.name =~ /^TEI/
      end

      def ignore_nodes(*nodes)
        @document.search(*nodes).each(&:remove)
        @document
      end

      alias_method :remove_nodes, :ignore_nodes

      private

      DUPLICATE_XML_VERSION = /<\?xml version=.*\?>\s*{2}/m
      def without_duplicate_declaration(doc)
        if doc.match(DUPLICATE_XML_VERSION)
          doc.sub(/^<\?xml version=.*\?>/, '')
        else
          doc
        end
      end

      def try_to_find_tei_root
        tei = @document.xpath('//*[name() = "TEI" or name() = "TEI.2"]').first
        if tei
          if RUBY_ENGINE == "jruby"
            # Pretty unnecessarily reparses the document fragment, but
            # nokogiri-java seems to have problems with handling namespaces
            # when assigning the root node of a document by hand.
            #
            # The issue has been reported on the nokogiri-talk Google group
            # and is described in detail there (currently awaiting approval)
{'fraction_non_alphanumeric': 0.10089781958101753, 'fraction_numerical': 0.0017101325352714834, 'mean_word_length': 2.08300395256917, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 4, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5883142', 'n_tokens_mistral': 754, 'n_tokens_neox': 724, 'n_words': 166}