<commit_before>require File.join(File.dirname(__FILE__), 'test_helper')

class NumerizerTest < TestCase
  def test_straight_parsing
    strings = {
      1 => 'one',
      5 => 'five',
      10 => 'ten',
      11 => 'eleven',
      12 => 'twelve',
      13 => 'thirteen',
      14 => 'fourteen',
      15 => 'fifteen',
      16 => 'sixteen',
      17 => 'seventeen',
      18 => 'eighteen',
      19 => 'nineteen',
      20 => 'twenty',
      27 => 'twenty seven',
      31 => 'thirty-one',
      37 => 'thirty-seven',
      41 => 'forty one',
      42 => 'fourty two',
      59 => 'fifty nine',
      100 => 'one hundred',
      150 => 'one hundred and fifty',
      # 150 => 'one fifty',
      200 => 'two-hundred',
      500 => '5 hundred',
      999 => 'nine hundred and ninety nine',
      1_000 => 'one thousand',
      1_200 => 'twelve hundred',
      17_000 => 'seventeen thousand',
      21_473 => 'twentyone-thousand-four-hundred-and-seventy-three',
      74_002 => 'seventy four thousand and two',
      99_999 => 'ninety nine thousand nine hundred ninety nine',
      100_000 => '100 thousand',
      250_000 => 'two hundred fifty thousand',
      1_000_000 => 'one million',
      1_250_007 => 'one million two hundred fifty thousand and seven',
      1_000_000_000 => 'one billion',
      1_000_000_001 => 'one billion and one'
    }

    strings.keys.sort.each do |key|
      assert_equal key, Numerizer.numerize(strings[key]).to_i
    end

    assert_equal "2.5", Numerizer.numerize("two and a half")
    assert_equal "1/2", Numerizer.numerize("one half")
    assert_equal "100", Numerizer.numerize("a hundred")
    assert_equal "100", Numerizer.numerize("hundred a")
    assert_equal "1200", Numerizer.numerize('one thousand two hundred')
  end

  def test_combined_double_digets
    assert_equal "21", Numerizer.numerize("twentyone")
    assert_equal "37", Numerizer.numerize("thirtyseven")
  end

  def test_fractions_in_words
    assert_equal "1/4", Numerizer.numerize("1 quarter")
    assert_equal "1/4", Numerizer.numerize("one quarter")<commit_msg>refactor tests<commit_after>require File.join(File.dirname(__FILE__), 'test_helper')

class NumerizerTest < TestCase
  def test_straight_parsing
    strings = {
      1 => 'one',
      5 => 'five',
      10 => 'ten',
      11 => 'eleven',
      12 => 'twelve',
      13 => 'thirteen',
      14 => 'fourteen',
      15 => 'fifteen',
      16 => 'sixteen',
      17 => 'seventeen',
      18 => 'eighteen',
      19 => 'nineteen',
      20 => 'twenty',
      27 => 'twenty seven',
      31 => 'thirty-one',
      37 => 'thirty-seven',
      41 => 'forty one',
      42 => 'fourty two',
      59 => 'fifty nine',
      100 => ['one hundred', 'a hundred', 'hundred a'],
      150 => 'one hundred and fifty',
      # 150 => 'one fifty',
      200 => 'two-hundred',
      500 => '5 hundred',
      999 => 'nine hundred and ninety nine',
      1_000 => 'one thousand',
      1_200 => ['twelve hundred', 'one thousand two hundred'],
      17_000 => 'seventeen thousand',
      21_473 => 'twentyone-thousand-four-hundred-and-seventy-three',
      74_002 => 'seventy four thousand and two',
      99_999 => 'ninety nine thousand nine hundred ninety nine',
      100_000 => '100 thousand',
      250_000 => 'two hundred fifty thousand',
      1_000_000 => 'one million',
      1_250_007 => 'one million two hundred fifty thousand and seven',
      1_000_000_000 => 'one billion',
      1_000_000_001 => 'one billion and one'
    }

    strings.sort.each do |key, value|
      Array(value).each do |value|
        assert_equal key, Numerizer.numerize(value).to_i
      end
    end

    assert_equal "2.5", Numerizer.numerize("two and a half")
    assert_equal "1/2", Numerizer.numerize("one half")
  end

  def test_combined_double_digets
    assert_equal "21", Numerizer.numerize("twentyone")
    assert_equal "37", Numerizer.numerize("thirtyseven")
  end

  def test_fractions_in_words
    assert_equal "1/4", Numerizer.numerize("1 quarter")
    assert_equal "1/4", Numerizer.numerize("one quarter")
{'fraction_non_alphanumeric': 0.14486326681448633, 'fraction_numerical': 0.07390983000739099, 'mean_word_length': 2.745387453874539, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1742724', 'n_tokens_mistral': 1526, 'n_tokens_neox': 1317, 'n_words': 353}