<commit_before># encoding: UTF-8

require "test/unit"

$: << File.join(File.expand_path(File.dirname(__FILE__)), '../lib')
require File.join(File.expand_path(File.dirname(__FILE__)), "../init.rb")

class StringExtensionsTest < Test::Unit::TestCase
  def test_to_html
    require "rubygems"
    require "RedCloth"
    {
      "h1. A Solution" => "<h1>A Solution</h1>",
      "I hated wrapping textilize around a string.\n\nIt always felt dirty." =>
        "<p>I hated wrapping textilize around a string.</p>\n<p>It always felt dirty.</p>",
      "I think _this_ is awesome" => "<p>I think <em>this</em> is awesome</p>",
      "Um... _*really*_, man" => "<p>Um&#8230; <em><strong>really</strong></em>, man</p>"
    }.each do |plain, html|
      assert_equal html, plain.to_html
    end
  rescue LoadError
    puts "\n>> Could not load RedCloth. String#to_html was not tested.\n>> Please gem install RedCloth if you'd like to use this functionality."
  end

  def test_to_html_lite
    require "rubygems"
    require "RedCloth"
    {
      "I have no pee on me" => "I have no pee on me",
      "But I _do_ get Textile!" => "But I <em>do</em> get Textile!"
    }.each do |plain, html|
      assert_equal html, plain.to_html(:lite)
    end
  rescue LoadError
    puts "\n>> Could not load RedCloth. String#to_html (with :lite argument) was not tested.\n>> Please gem install RedCloth if you'd like to use this functionality."
  end

  def test_to_url
    {
      "<p>This has 100% too much    <em>formatting</em></p>" =>
        "this-has-100-percent-too-much-formatting",
      "Tea   &amp; crumpets &amp; <strong>cr&ecirc;pes</strong> for me!" =>
        "tea-and-crumpets-and-crepes-for-me",
      "The Suspense... Is... Killing Me" =>
        "the-suspense-dot-dot-dot-is-dot-dot-dot-killing-me",
      "How to use attr_accessible and attr_protected" =>
        "how-to-use-attr-accessible-and-attr-protected",
      "I'm just making sure there's nothing wrong with things!" =>
        "im-just-making-sure-theres-nothing-wrong-with-things",
      "foo = bar and bar=foo" =>
        "foo-equals-bar-and-bar-equals-foo",
      "Period.period" =>
        "period-dot-period",
      "Will…This Work?" =>
        "will-dot-dot-dot-this-work",
      "We check the D.N.A for matches" =>
        "we-check-the-dna-for-matches",
      "We check the D.N.A. for matches" =>
        "we-check-the-dna-for-matches",
      "Go to the Y.M.C.America" =>
        "go-to-the-ymcamerica",
      "He shops at J.C. Penney" =>
        "he-shops-at-jc-penney",
      "¼ pound with cheese" =>
        "one-fourth-pound-with-cheese",
      "<NAME>" =>
        "wills-ferrel",
      "Капитал" =>
        "kapital",
      "Ελλάδα" =>
        "ellada",
      "中文" =>
        "zhong-wen",
      "<NAME>" =>
        "paul-cezanne",
      "21'17ʼ51" =>
        "21-17-51",
      "ITCZ÷1 (21°17ʼ51.78”N / 89°35ʼ28.18”O / 26-04-08 / 09:00 am)" =>
        "itcz-divide-1-21-degrees-17-51-dot-78-n-slash-89-degrees-35-28-dot-18-o-slash-26-04-08-slash-09-00-am",
      "／" =>
        "slash",
      "私はガラスを食べられます。それは私を傷つけません。" =>
        "si-hagarasuwoshi-beraremasu-sorehasi-woshang-tukemasen",
      "ǝ is a magical string" =>
        "at-is-a-magical-string",
      "either | or" =>
        "either-or",
      "La Maison d`Uliva" =>
        "la-maison-duliva",
    }.each do |html, plain|
      assert_equal plain, html.to_url
    end
  end

  def tests_that_only_pass_on_travis_for_1_9
    if RUBY_VERSION.include?('1.9')
      {
        "カッページ・テラスに日系カフェ＆バー、店内にDJブースも - シンガポール経済新聞" =>
          "katupeziterasuniri-xi-kahue-and-ba-dian-nei-nidjbusumo-singaporujing-ji-xin-wen",
        "AVアンプ、ホームシアターシステム、スピーカーシステム等" =>
          "avanpu-homusiatasisutemu-supikasisutemudeng",
        "У лукоморья дуб зеленый" =>
          "u-lukomoria-dub-zielienyi"
      }.each do |html, plain|
        assert_equal plain, html.to_url
      end
    else
      message = "\n-----\n"
      message << "Skipping a few transliteration tests that for some reason fail on Travis-CI for the latest 1.8.7, which I cannot install for some reason on my Mac. "
      message << "If you can run these tests AND they fail for you AND you care to help debug what the problem is, please let me know at sconds<EMAIL>. Thanks."
      message << "\n-----"
      puts message
    end
  end

  def test_to_url_with_excludes
    assert_equal "So Fucking Special", "So Fucking Special".to_url(:exclude => "So Fucking Special")
  end

  def test_remove_formatting
    {
      "<p>This has 100% too much    <em>formatting</em></p>" =>
        "This has 100 percent too much formatting",
      "Tea   &amp; crumpets &amp; <strong>cr&ecirc;pes</strong> for me!" =>
        "Tea and crumpets and crepes for me"
    }.each do |html, plain|
      assert_equal plain, html.remove_formatting
    end
  end

  def test_strip_html_tags
    {
      "<h1><em>This</em> is good but <strong>that</strong> is better</h1>" =>
        "This is good but that is better",
      "<p>This is invalid XHTML but valid HTML, right?" =>
        "This is invalid XHTML but valid HTML, right?",
      "<p class='foo'>Everything goes!</p>" => "Everything goes!",
      "<ol>This is completely invalid and just plain wrong</p>" =><commit_msg>reinstate 1.8.7 troublemakers in the regular test suite. hat tip [if it works] to nathan sutton.<commit_after># encoding: UTF-8
$KCODE = "U"

require "test/unit"

$: << File.join(File.expand_path(File.dirname(__FILE__)), '../lib')
require File.join(File.expand_path(File.dirname(__FILE__)), "../init.rb")

class StringExtensionsTest < Test::Unit::TestCase
  def test_to_html
    require "rubygems"
    require "RedCloth"
    {
      "h1. A Solution" => "<h1>A Solution</h1>",
      "I hated wrapping textilize around a string.\n\nIt always felt dirty." =>
        "<p>I hated wrapping textilize around a string.</p>\n<p>It always felt dirty.</p>",
      "I think _this_ is awesome" => "<p>I think <em>this</em> is awesome</p>",
      "Um... _*really*_, man" => "<p>Um&#8230; <em><strong>really</strong></em>, man</p>"
    }.each do |plain, html|
      assert_equal html, plain.to_html
    end
  rescue LoadError
    puts "\n>> Could not load RedCloth. String#to_html was not tested.\n>> Please gem install RedCloth if you'd like to use this functionality."
  end

  def test_to_html_lite
    require "rubygems"
    require "RedCloth"
    {
      "I have no pee on me" => "I have no pee on me",
      "But I _do_ get Textile!" => "But I <em>do</em> get Textile!"
    }.each do |plain, html|
      assert_equal html, plain.to_html(:lite)
    end
  rescue LoadError
    puts "\n>> Could not load RedCloth. String#to_html (with :lite argument) was not tested.\n>> Please gem install RedCloth if you'd like to use this functionality."
  end

  def test_to_url
    {
      "<p>This has 100% too much    <em>formatting</em></p>" =>
        "this-has-100-percent-too-much-formatting",
      "Tea   &amp; crumpets &amp; <strong>cr&ecirc;pes</strong> for me!" =>
        "tea-and-crumpets-and-crepes-for-me",
      "The Suspense... Is... Killing Me" =>
        "the-suspense-dot-dot-dot-is-dot-dot-dot-killing-me",
      "How to use attr_accessible and attr_protected" =>
        "how-to-use-attr-accessible-and-attr-protected",
      "I'm just making sure there's nothing wrong with things!" =>
        "im-just-making-sure-theres-nothing-wrong-with-things",
      "foo = bar and bar=foo" =>
        "foo-equals-bar-and-bar-equals-foo",
      "Period.period" =>
        "period-dot-period",
      "Will…This Work?" =>
        "will-dot-dot-dot-this-work",
      "We check the D.N.A for matches" =>
        "we-check-the-dna-for-matches",
      "We check the D.N.A. for matches" =>
        "we-check-the-dna-for-matches",
      "Go to the Y.M.C.America" =>
        "go-to-the-ymcamerica",
      "He shops at J.C. Penney" =>
        "he-shops-at-jc-penney",
      "¼ pound with cheese" =>
        "one-fourth-pound-with-cheese",
      "<NAME>" =>
        "wills-ferrel",
      "Капитал" =>
        "kapital",
      "Ελλάδα" =>
        "ellada",
      "中文" =>
        "zhong-wen",
      "<NAME>" =>
        "paul-cezanne",
      "21'17ʼ51" =>
        "21-17-51",
      "ITCZ÷1 (21°17ʼ51.78”N / 89°35ʼ28.18”O / 26-04-08 / 09:00 am)" =>
        "itcz-divide-1-21-degrees-17-51-dot-78-n-slash-89-degrees-35-28-dot-18-o-slash-26-04-08-slash-09-00-am",
      "／" =>
        "slash",
      "私はガラスを食べられます。それは私を傷つけません。" =>
        "si-hagarasuwoshi-beraremasu-sorehasi-woshang-tukemasen",
      "ǝ is a magical string" =>
        "at-is-a-magical-string",
      "either | or" =>
        "either-or",
      "La Maison d`Uliva" =>
        "la-maison-duliva",
      "カッページ・テラスに日系カフェ＆バー、店内にDJブースも - シンガポール経済新聞" =>
        "katupeziterasuniri-xi-kahue-and-ba-dian-nei-nidjbusumo-singaporujing-ji-xin-wen",
      "AVアンプ、ホームシアターシステム、スピーカーシステム等" =>
        "avanpu-homusiatasisutemu-supikasisutemudeng",
      "У лукоморья дуб зеленый" =>
        "u-lukomoria-dub-zielienyi"
    }.each do |html, plain|
      assert_equal plain, html.to_url
    end
  end

  def test_to_url_with_excludes
    assert_equal "So Fucking Special", "So Fucking Special".to_url(:exclude => "So Fucking Special")
  end

  def test_remove_formatting
    {
      "<p>This has 100% too much    <em>formatting</em></p>" =>
        "This has 100 percent too much formatting",
      "Tea   &amp; crumpets &amp; <strong>cr&ecirc;pes</strong> for me!" =>
        "Tea and crumpets and crepes for me"
    }.each do |html, plain|
      assert_equal plain, html.remove_formatting
    end
  end

  def test_strip_html_tags
    {
      "<h1><em>This</em> is good but <strong>that</strong> is better</h1>" =>
        "This is good but that is better",
      "<p>This is invalid XHTML but valid HTML, right?" =>
        "This is invalid XHTML but valid HTML, right?",
      "<p class='foo'>Everything goes!</p>" => "Everything goes!",
      "<ol>This is completely invalid and just plain wrong</p>" =>
{'fraction_non_alphanumeric': 0.15440881763527053, 'fraction_numerical': 0.018637274549098196, 'mean_word_length': 2.5193935119887163, 'pattern_counts': {'":': 0, '<': 102, '<?xml version=': 0, '>': 188, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1054377', 'n_tokens_mistral': 3897, 'n_tokens_neox': 3722, 'n_words': 865}