<commit_before>module Docs
  class Git
    class CleanHtmlFilter < Filter
      def call
        root_page? ? root : other
        doc
      end

      def root
        at_css('h1').content = 'Git'
      end

      def other
        css('h1 + h2', '#_git + div', '#_git').remove

        css('> div', 'pre > tt', 'pre > em', 'div.paragraph').each do |node|
          node.before(node.children).remove
        end

        css('> h1').each do |node|
          node.content = node.content.remove(/\(\d\) Manual Page/)
        end

        unless at_css('> h1')
          doc.child.before("<h1>#{slug}</h1>")
        end

        unless at_css('> h2')
          css('> h3').each do |node|
            node.name = 'h2'
          end
        end

        css('h2').each do |node|
          node.content = node.content.capitalize
        end
<commit_msg>fixed links in git<commit_after>module Docs
  class Git
    class CleanHtmlFilter < Filter

      def call
        root_page? ? root : other
        doc
      end

      def root
        at_css('h1').content = 'Git'
      end

      def other
        css('h1 + h2', '#_git + div', '#_git').remove

        css('a[href]').each do |node|
          if !node['href'].start_with? 'http://' and !node['href'].start_with? 'https://'
            if node['href'].start_with? ':'
              node['class'] = 'broken'
              node['href'] = '/help#brokenlink'
            elsif node['href'].start_with? 'howto/'
              node['href'] = 'https://github.com/git/git/blob/master/Documentation/' + node['href'] + '.txt'
            else
              sluglist = slug.split('/')
              nodelist = node['href'].split('/')
              newhref = []
              nodelist.each do |item|
                if item == '..'
                  sluglist.pop
                else
                  newhref << item
                end
              end
              sluglist.pop
              if sluglist.size>0
                node['href'] = sluglist.join('/') + '/' + newhref.join('/')
              else
                node['href'] = newhref.join('/')
              end
            end
          end
        end

        css('> div', 'pre > tt', 'pre > em', 'div.paragraph').each do |node|
          node.before(node.children).remove
        end

        css('> h1').each do |node|
          node.content = node.content.remove(/\(\d\) Manual Page/)
        end

        unless at_css('> h1')
          doc.child.before("<h1>#{slug}</h1>")
        end

        unless at_css('> h2')
          css('> h3').each do |node|
            node.name = 'h2'
          end
        end

        css('h2').each do |node|
          node.content = node.content.capitalize
        end

{'fraction_non_alphanumeric': 0.15027624309392265, 'fraction_numerical': 0.00847145488029466, 'mean_word_length': 1.3741258741258742, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 22, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3084833', 'n_tokens_mistral': 839, 'n_tokens_neox': 806, 'n_words': 191}