<commit_before>require File.join(File.dirname(__FILE__), 'helper')

class TestWordToMarkdown < Test::Unit::TestCase

  should "properly parse italic text" do
    validate_fixture "em", "This word is _italic_."
  end

  should "properly parse bold text" do
    validate_fixture "strong", "This word is **bold**."
  end

  should "properly parse explicit h1's" do
    validate_fixture "h1", "# Heading 1\n\nParagraph text"
  end

  should "properly parse explicit h2's" do
    validate_fixture "h2", "## Heading 2\n\nParagraph text"
  end

  should "properly parse paragraphs" do
    validate_fixture "p", "This is paragraph text."
  end

  should "properly parse multiple-headings" do
    expected = "# H1\n\nParagraph\n\n## H2\n\nParagraph\n\n### H3\n\nParagraph"
    validate_fixture "multiple-headings", expected
  end

  should "properly parse unnumbered lists" do
    validate_fixture "ul", "- One\n- Two\n- Three"
  end

  should "properly parse numbered lists" do
    validate_fixture "ol", "1. One\n\n2. Two\n\n3. Three"
  end

  should "accept string input" do
    assert_equal "# Heading", WordToMarkdown.new("<h1>Heading</h1>").to_s
  end


end<commit_msg>test for encoding<commit_after>require File.join(File.dirname(__FILE__), 'helper')

class TestWordToMarkdown < Test::Unit::TestCase

  should "properly parse italic text" do
    validate_fixture "em", "This word is _italic_."
  end

  should "properly parse bold text" do
    validate_fixture "strong", "This word is **bold**."
  end

  should "properly parse explicit h1's" do
    validate_fixture "h1", "# Heading 1\n\nParagraph text"
  end

  should "properly parse explicit h2's" do
    validate_fixture "h2", "## Heading 2\n\nParagraph text"
  end

  should "properly parse paragraphs" do
    validate_fixture "p", "This is paragraph text."
  end

  should "properly parse multiple-headings" do
    expected = "# H1\n\nParagraph\n\n## H2\n\nParagraph\n\n### H3\n\nParagraph"
    validate_fixture "multiple-headings", expected
  end

  should "properly parse unnumbered lists" do
    validate_fixture "ul", "- One\n- Two\n- Three"
  end

  should "properly parse numbered lists" do
    validate_fixture "ol", "1. One\n\n2. Two\n\n3. Three"
  end

  should "accept string input" do
    assert_equal "# Heading", WordToMarkdown.new("<h1>Heading</h1>").to_s
  end

  should "not mangle encoding" do
    doc = WordToMarkdown.new "<span>’</span>"
    assert_equal "’", doc.to_s
  end
end
{'fraction_non_alphanumeric': 0.11867862969004894, 'fraction_numerical': 0.011419249592169658, 'mean_word_length': 3.6992337164750957, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7033088', 'n_tokens_mistral': 887, 'n_tokens_neox': 834, 'n_words': 244}