<commit_before>class HTMLEntities
  class Decoder #:nodoc:
    def initialize(flavor)
      @flavor = flavor
    end

    def decode(source)
      source.to_s.gsub(named_entity_regexp) {
        (cp = map[$1]) ? [cp].pack('U') : $&
      }.gsub(/&#([0-9]{1,7});|&#x([0-9a-f]{1,6});/i) {
        $1 ? [$1.to_i].pack('U') : [$2.to_i(16)].pack('U')
      }
    end

  private
    def map
      @map ||= HTMLEntities::MAPPINGS[@flavor]
    end

    def named_entity_regexp
      @named_entity_regexp ||= (
        min_length = map.keys.map{ |a| a.length }.min
        max_length = map.keys.map{ |a| a.length }.max
        ok_chars = @flavor.to_s == 'expanded' ? '(?:b\.)?[a-z][a-z0-9]' : '[a-z][a-z0-9]'
        /&(#{ok_chars}{#{min_length-1},#{max_length-1}});/i
      )
    end
  end
end<commit_msg>Minor simplification of non-time-critical code.<commit_after>class HTMLEntities
  class Decoder #:nodoc:
    def initialize(flavor)
      @flavor = flavor
    end

    def decode(source)
      source.to_s.gsub(named_entity_regexp) {
        (cp = map[$1]) ? [cp].pack('U') : $&
      }.gsub(/&#([0-9]{1,7});|&#x([0-9a-f]{1,6});/i) {
        $1 ? [$1.to_i].pack('U') : [$2.to_i(16)].pack('U')
      }
    end

  private
    def map
      @map ||= HTMLEntities::MAPPINGS[@flavor]
    end

    def named_entity_regexp
      @named_entity_regexp ||= (
        key_lengths = map.keys.map{ |k| k.length }
        ok_chars = @flavor.to_s == 'expanded' ? '(?:b\.)?[a-z][a-z0-9]' : '[a-z][a-z0-9]'
        /&(#{ ok_chars }{#{ key_lengths.min - 1 },#{ key_lengths.max - 1 }});/i
      )
    end
  end
end
{'fraction_non_alphanumeric': 0.21810182275298554, 'fraction_numerical': 0.0251414204902577, 'mean_word_length': 2.4089935760171306, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2615894', 'n_tokens_mistral': 692, 'n_tokens_neox': 676, 'n_words': 100}