<commit_before>require 'rubyXL/objects/ooxml_object'
require 'rubyXL/objects/simple_types'
require 'rubyXL/objects/container_nodes'
require 'rubyXL/objects/color'

module RubyXL

  # http://www.datypic.com/sc/ooxml/e-ssml_t-1.html
  class Text < OOXMLObject
    define_attribute(:_,           :string, :accessor => :value)
    define_attribute(:'xml:space', %w{ preserve })
    define_element_name 't'

    # http://www.w3.org/TR/REC-xml/#NT-Char:
    # Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]

    INVALID_XML10_CHARS = /([^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD\u10000-\u10FFFF])/
    ESCAPED_UNICODE = /_x([0-9A-F]{4})_/

    def before_write_xml
      preserve_whitespace
      self.value.gsub!(INVALID_XML10_CHARS) { |c| "_x%04x_" % c.ord }
      true
    end

    def to_s
      value.to_s.gsub(ESCAPED_UNICODE) { |m| $1.hex.chr(::Encoding::UTF_8) }
    end
  end<commit_msg>Fix wrong escaping of U+10000 to U+10FFFF range<commit_after>require 'rubyXL/objects/ooxml_object'
require 'rubyXL/objects/simple_types'
require 'rubyXL/objects/container_nodes'
require 'rubyXL/objects/color'

module RubyXL

  # http://www.datypic.com/sc/ooxml/e-ssml_t-1.html
  class Text < OOXMLObject
    define_attribute(:_,           :string, :accessor => :value)
    define_attribute(:'xml:space', %w{ preserve })
    define_element_name 't'

    # http://www.w3.org/TR/REC-xml/#NT-Char:
    # Char ::= #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]

    INVALID_XML10_CHARS = /([^\x09\x0A\x0D\x20-\uD7FF\uE000-\uFFFD\u{10000}-\u{10FFFF}])/
    ESCAPED_UNICODE = /_x([0-9A-F]{4})_/

    def before_write_xml
      preserve_whitespace
      self.value.gsub!(INVALID_XML10_CHARS) { |c| "_x%04x_" % c.ord }
      true
    end

    def to_s
      value.to_s.gsub(ESCAPED_UNICODE) { |m| $1.hex.chr(::Encoding::UTF_8) }
    end
  end
{'fraction_non_alphanumeric': 0.1695095948827292, 'fraction_numerical': 0.05063965884861407, 'mean_word_length': 3.305045871559633, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6784310', 'n_tokens_mistral': 878, 'n_tokens_neox': 782, 'n_words': 108}