<commit_before>require 'nokogiri'

module Jekyll
  module AtomFeedFilters
    def strip_html_attrs(html)
      doc = Nokogiri::HTML.fragment(html)
      doc.xpath('@style|.//@style|@data-lang|.//@data-lang|@controls|.//@controls').remove
      doc.to_s
    end

    # Matches all whitespace that follows
    #   1. A '>', which closes an XML tag or
    #   2. A '}', which closes a Liquid tag
    # We will strip all of this whitespace to minify the template
    #
    # This isn't as nice as using a proper XML parser, but I've been unable
    # to get that working in Ruby!  This is taken from jekyll-feed.
    MINIFY_REGEX = %r!(?<=>|})\s+!

    def minify_xml(xml)
      xml.gsub(MINIFY_REGEX, "")
    end
  end
end

Liquid::Template::register_filter(Jekyll::AtomFeedFilters)<commit_msg>Document and tidy up atom_feeds.rb<commit_after>require 'nokogiri'

module Jekyll
  module AtomFeedFilters

    # Remove inline attributes from an HTML string that aren't allowed in
    # an Atom feed, according to https://github.com/rubys/feedvalidator
    #
    # Params:
    # +html+:: HTML string to clean.
    #
    def strip_html_attrs(html)
      doc = Nokogiri::HTML.fragment(html)
      doc.xpath('@style|.//@style|@data-lang|.//@data-lang|@controls|.//@controls').remove
      doc.to_s
    end

    # Minify an XML string.
    #
    # Here minification just means removing all whitespace that comes
    # after a '>'.  This isn't as nice as using a proper XML parser, but
    # I've been unable to get that working in Ruby.
    # TODO: Use Nokogiri to do this properly.
    #
    # Params:
    # +xml+:: XML string to minify.
    #
    def minify_xml(xml)
      xml.gsub(%r|>\s+|, ">")
    end

  end
end

Liquid::Template::register_filter(Jekyll::AtomFeedFilters)
{'fraction_non_alphanumeric': 0.11784702549575071, 'fraction_numerical': 0.0011331444759206798, 'mean_word_length': 2.9070796460176993, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 8, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 7}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4862253', 'n_tokens_mistral': 613, 'n_tokens_neox': 602, 'n_words': 172}