<commit_before>require 'html/pipeline'

class PageTocFilter < HTML::Pipeline::Filter

  def call
    doc.search('p').each do |node|
      content = node.to_html
      next unless content == '<p>{:toc}</p>'
      html = page_toc_filter(doc)
      node.replace(html)
    end

    doc
  end

  def page_toc_filter(doc)
    toc = ''
    levels = context[:toc_levels] || 'h2'

    doc.search(levels).each do |node|
      text = node.text
      id = node.child['id']
      toc << %(<li><a href="##{id}" id="markdown-toc-#{id}">#{text}</a></li>\n)
    end
    toc = %(<ul id="markdown-toc">\n#{toc}</ul>) unless toc.empty?
    toc
  end
end<commit_msg>Make nested ToC list<commit_after>require 'html/pipeline'

class PageTocFilter < HTML::Pipeline::Filter

  def call
    doc.search('p').each do |node|
      content = node.to_html
      next unless content == '<p>{:toc}</p>'
      html = page_toc_filter(doc)
      node.replace(html)
    end

    doc
  end

  def page_toc_filter(doc)
    levels = doc.search(context[:toc_levels] || 'h2')
    return "" if levels.empty?

    toc = %(<ul id="markdown-toc">\n)
    last_level = nil

    levels.each do |node|
      current_level = node.name.match(/h(\d)/)[1]

      text = node.text
      id = node.child['id']

      link = %(<a href="##{id}" id="markdown-toc-#{id}">#{text}</a>)

      if last_level.nil?
        toc << %(<li>\n#{link}\n)
      elsif current_level == last_level
        toc << %(</li><li>\n#{link}\n)
      elsif current_level > last_level
        toc << %(<ul><li>#{link}\n)
      elsif current_level < last_level
        toc << %(</li></ul><li>#{link}\n)
      end

      last_level = current_level
    end
    toc << %(</li></ul>)
    toc
  end
end
{'fraction_non_alphanumeric': 0.18213660245183888, 'fraction_numerical': 0.0017513134851138354, 'mean_word_length': 2.5634095634095635, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3588120', 'n_tokens_mistral': 654, 'n_tokens_neox': 614, 'n_words': 126}