<commit_before>module Hexp
  class Unparser
    include Adamantium

    APOS   = ?'.freeze
    QUOT   = ?".freeze
    LT     = '<'.freeze
    GT     = '>'.freeze
    SPACE  = ' '.freeze
    EQ     = '='.freeze
    AMP    = '&'.freeze
    FSLASH = '/'.freeze

    E_AMP  = '&amp;'.freeze
    E_APOS = '&#x27;'.freeze
    E_QUOT = '&quot;'.freeze
    E_LT   = '&lt;'.freeze
    E_GT   = '&gt;'.freeze

    ESCAPE_ATTR_APOS = {AMP => E_AMP, APOS => E_APOS}
    ESCAPE_ATTR_QUOT = {AMP => E_AMP, QUOT => E_QUOT}
    ESCAPE_TEXT      = {AMP => E_AMP, APOS => E_APOS, QUOT => E_QUOT, LT => E_LT, GT => E_GT}

    ESCAPE_ATTR_APOS_REGEX = Regexp.new("[#{ESCAPE_ATTR_APOS.keys.join}]")
    ESCAPE_ATTR_QUOT_REGEX = Regexp.new("[#{ESCAPE_ATTR_QUOT.keys.join}]")
    ESCAPE_TEXT_REGEX      = Regexp.new("[#{ESCAPE_TEXT.keys.join}]")

    DEFAULT_OPTIONS = {
      encoding: Encoding.default_external,
      no_escape: [:script],
      void: [ :area, :base, :br, :col, :command, :embed, :hr, :img, :input,
              :keygen, :link, :meta, :param, :source, :track, :wbr ]
    }

    attr_reader :options

    def initialize(options)
      @options = DEFAULT_OPTIONS.merge(options)
    end

    def call(node)
      buffer = String.new.force_encoding(options[:encoding])
      add_node(buffer, node)
      buffer.freeze
    end

    private

    def add_node(buffer, node)
      if node.text?
        buffer << escape_text(node)
      else
        add_tag(buffer, node.tag, node.attributes, node.children)
      end
    end
<commit_msg>Bring back the :html5 option<commit_after>module Hexp
  class Unparser
    include Adamantium

    APOS   = ?'.freeze
    QUOT   = ?".freeze
    LT     = '<'.freeze
    GT     = '>'.freeze
    SPACE  = ' '.freeze
    EQ     = '='.freeze
    AMP    = '&'.freeze
    FSLASH = '/'.freeze

    E_AMP  = '&amp;'.freeze
    E_APOS = '&#x27;'.freeze
    E_QUOT = '&quot;'.freeze
    E_LT   = '&lt;'.freeze
    E_GT   = '&gt;'.freeze

    ESCAPE_ATTR_APOS = {AMP => E_AMP, APOS => E_APOS}
    ESCAPE_ATTR_QUOT = {AMP => E_AMP, QUOT => E_QUOT}
    ESCAPE_TEXT      = {AMP => E_AMP, APOS => E_APOS, QUOT => E_QUOT, LT => E_LT, GT => E_GT}

    ESCAPE_ATTR_APOS_REGEX = Regexp.new("[#{ESCAPE_ATTR_APOS.keys.join}]")
    ESCAPE_ATTR_QUOT_REGEX = Regexp.new("[#{ESCAPE_ATTR_QUOT.keys.join}]")
    ESCAPE_TEXT_REGEX      = Regexp.new("[#{ESCAPE_TEXT.keys.join}]")

    DEFAULT_OPTIONS = {
      encoding: Encoding.default_external,
      no_escape: [:script],
      void: [ :area, :base, :br, :col, :command, :embed, :hr, :img, :input,
              :keygen, :link, :meta, :param, :source, :track, :wbr ]
    }

    attr_reader :options

    def initialize(options)
      @options = DEFAULT_OPTIONS.merge(options)
    end

    def call(node)
      buffer = String.new.force_encoding(options[:encoding])
      buffer << "<!DOCTYPE html>\n" if @options[:html5]
      add_node(buffer, node)
      buffer.freeze
    end

    private

    def add_node(buffer, node)
      if node.text?
        buffer << escape_text(node)
      else
        add_tag(buffer, node.tag, node.attributes, node.children)
      end
    end

{'fraction_non_alphanumeric': 0.14944, 'fraction_numerical': 0.00192, 'mean_word_length': 2.504484304932735, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3265419', 'n_tokens_mistral': 1260, 'n_tokens_neox': 1218, 'n_words': 231}