<commit_before>module GitHub::HTML
  module PlainTextFilter
    def self.call(body, context={})
      Pipeline.call(body, context)
    end

    class Filter < ::GitHub::HTML::Filter
      def call
        links = []

        doc.children.first.children.each do |node|
          if !%w[text a].include?(node.name)
            node.replace Nokogiri::XML::Text.new(node.to_s, doc)
          end
        end

        doc.search("a").each do |node|
          links << File.join(GitHub::SSLHost, node["href"])

          index = links.size
          label = "#{node.text} [#{index}]"

          node.replace(Nokogiri::XML::Text.new(label, doc))
        end

        if links.any?
          doc.add_child(Nokogiri::XML::Text.new("\n\n", doc))
        end

        links.each_with_index do |link, i|
          text = "[#{i + 1}]: #{link}\n"
          doc.add_child(Nokogiri::XML::Text.new(text, doc))
        end
      end
    end

    Pipeline = ::GitHub::HTML::Pipeline.new [
      lambda { |doc, _| "<p>#{doc}</p>" },
      ::GitHub::HTML::IssueMentionFilter,
      ::GitHub::HTML::CommitMentionFilter,
      Filter,
      lambda { |doc, _| doc.text }
    ]
  end
end<commit_msg>HTML parsers parse HTML, not text<commit_after>module GitHub::HTML
  module PlainTextFilter
    def self.call(body, context={})
      Pipeline.call(body, context)
    end

    class Filter < ::GitHub::HTML::Filter
      def call
        links = []

        doc.children.first.children.each do |node|
          if !%w[text a].include?(node.name)
            node.replace Nokogiri::XML::Text.new(node.to_s, doc)
          end
        end

        doc.search("a").each do |node|
          links << File.join(GitHub::SSLHost, node["href"])

          index = links.size
          label = "#{node.text} [#{index}]"

          node.replace(Nokogiri::XML::Text.new(label, doc))
        end

        if links.any?
          doc.add_child(Nokogiri::XML::Text.new("\n\n", doc))
        end

        links.each_with_index do |link, i|
          text = "[#{i + 1}]: #{link}\n"
          doc.add_child(Nokogiri::XML::Text.new(text, doc))
        end
      end
    end

    Pipeline = ::GitHub::HTML::Pipeline.new [
      lambda { |doc, _| "<p>#{EscapeUtils.escape_html(doc)}</p>" },
      ::GitHub::HTML::IssueMentionFilter,
      ::GitHub::HTML::CommitMentionFilter,
      Filter,
      lambda { |doc, _| doc.text }
    ]
  end
end
{'fraction_non_alphanumeric': 0.15963226076055162, 'fraction_numerical': 0.0008357709987463436, 'mean_word_length': 2.1541501976284585, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3935724', 'n_tokens_mistral': 822, 'n_tokens_neox': 790, 'n_words': 148}