<commit_before>class MarkdownFilter < TextFilter
  description_file File.dirname(__FILE__) + "/../markdown.html"
  def filter(text)
    if defined? RDiscount
      RDiscount.new(text, :smart).to_html
    else
      RubyPants.new(Kramdown::Document.new(text, {
        :auto_ids => auto_ids || false,
        :parse_block_html => parse_block_html || false,
        :coderay => nil
      }).to_html).to_html
    end
  end
end<commit_msg>set auto_ids and parse_block_html to false. closes #98<commit_after>class MarkdownFilter < TextFilter
  description_file File.dirname(__FILE__) + "/../markdown.html"
  def filter(text)
    if defined? RDiscount
      RDiscount.new(text, :smart).to_html
    else
      RubyPants.new(Kramdown::Document.new(text, {
        :auto_ids => false,
        :parse_block_html => false,
        :coderay => nil
      }).to_html).to_html
    end
  end
end
{'fraction_non_alphanumeric': 0.11604095563139932, 'fraction_numerical': 0.0022753128555176336, 'mean_word_length': 3.055299539170507, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2400455', 'n_tokens_mistral': 294, 'n_tokens_neox': 289, 'n_words': 58}