<commit_before>require 'helper'

class TestOnigiri < MiniTest::Unit::TestCase
  def setup
    
  end

  def test_normalized_text_has_no_punctuation
    text = "the, end."
    assert_equal "the end", Onigiri::Onigiri.normalize(text)
  end

  def test_tokenizes_text
    text = "one two"
    tokens = Onigiri::Onigiri.tokenize(text)
    assert_equal "one", tokens.first.name
    assert_equal "two", tokens.last.name
  end

  def test_selects_tagged_tokens_only
    tok_a = Onigiri::Token.new("10")
    tok_a.add_tag(:scalar)
    tok_b = Onigiri::Token.new("grams")
    assert_equal [tok_a], Onigiri::Onigiri.select_tagged_only([tok_a, tok_b])
  end

  def test_parsing_from_text_sclmsr_msr_ing
    Onigiri::Ingredient.set_ingredient 'cherry tomato'
    text = "10 lbs of cherry tomato"
    expected = {:ammount => 10, :ingredient => 'cherry tomato', :measurement => 'pound'}
    result = Onigiri::Onigiri.parse(text)
    assert_equal(expected[:ammount],      result[:ammount])
    assert_equal(expected[:ingredient],   result[:ingredient])
    assert_equal(expected[:measurement],  result[:measurement])
  end

  def test_parsing_from_text_including_ingredient_variation
    Onigiri::Ingredient.set_ingredient 'dijon mustard', 'dijjon'
    text = "1 tbsp Dijjon mustard"
    expected = {:ammount => 1, :ingredient => 'dijon mustard', :measurement => 'tablespoon'}
    result = Onigiri::Onigiri.parse(text)
    assert_equal(expected[:ammount],      result[:ammount])
    assert_equal(expected[:ingredient],   result[:ingredient])
    assert_equal(expected[:measurement],  result[:measurement])
  end

  def test_parsing_sclmsr_msr_ing
    text = "1 1/2 tbsp honey"
    expected = {:ammount => 1.5, :ingredient => 'honey', :measurement => 'tablespoon'}
    result = Onigiri::Onigiri.parse(text)
    assert_equal(expected[:ammount],      result[:ammount])
    assert_equal(expected[:ingredient],   result[:ingredient])
    assert_equal(expected[:measurement],  result[:measurement])
  end
end<commit_msg>test parsing of a string with multiple ingredients <commit_after>require 'helper'

class TestOnigiri < MiniTest::Unit::TestCase
  def setup
    
  end

  def test_normalized_text_has_no_punctuation
    text = "the, end."
    assert_equal "the end", Onigiri::Onigiri.normalize(text)
  end

  def test_tokenizes_text
    text = "one two"
    tokens = Onigiri::Onigiri.tokenize(text)
    assert_equal "one", tokens.first.name
    assert_equal "two", tokens.last.name
  end

  def test_selects_tagged_tokens_only
    tok_a = Onigiri::Token.new("10")
    tok_a.add_tag(:scalar)
    tok_b = Onigiri::Token.new("grams")
    assert_equal [tok_a], Onigiri::Onigiri.select_tagged_only([tok_a, tok_b])
  end

  def test_parsing_from_text_sclmsr_msr_ing
    Onigiri::Ingredient.set_ingredient 'cherry tomato'
    text = "10 lbs of cherry tomato"
    expected = {:ammount => 10, :ingredient => 'cherry tomato', :measurement => 'pound'}
    result = Onigiri::Onigiri.parse(text)
    assert_equal(expected[:ammount],      result[:ammount])
    assert_equal(expected[:ingredient],   result[:ingredient])
    assert_equal(expected[:measurement],  result[:measurement])
  end

  def test_parsing_from_text_including_ingredient_variation
    Onigiri::Ingredient.set_ingredient 'dijon mustard', 'dijjon'
    text = "1 tbsp Dijjon mustard"
    expected = {:ammount => 1, :ingredient => 'dijon mustard', :measurement => 'tablespoon'}
    result = Onigiri::Onigiri.parse(text)
    assert_equal(expected[:ammount],      result[:ammount])
    assert_equal(expected[:ingredient],   result[:ingredient])
    assert_equal(expected[:measurement],  result[:measurement])
  end

  def test_parsing_sclmsr_msr_ing
    text = "1 1/2 tbsp honey"
    expected = {:ammount => 1.5, :ingredient => 'honey', :measurement => 'tablespoon'}
    result = Onigiri::Onigiri.parse(text)
    assert_equal(expected[:ammount],      result[:ammount])
    assert_equal(expected[:ingredient],   result[:ingredient])
    assert_equal(expected[:measurement],  result[:measurement])
  end

  def test_parsing_multiple_ingredients_
    text = "1/2 cup each carrots, celery and onions"
    assert_equal result('carrot', 0.5, 'cup'),  Onigiri::Onigiri.parse(text)
  end

  def result(ingredient, ammount="", measurement="", modifier="")
     result = {  :ammount => ammount, 
                 :ingredient => ingredient,
                 :measurement => measurement, 
                 :modifier => modifier 
               }
  end
end
{'fraction_non_alphanumeric': 0.12337952615109522, 'fraction_numerical': 0.006705409029950827, 'mean_word_length': 4.308422301304864, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6471360', 'n_tokens_mistral': 1577, 'n_tokens_neox': 1501, 'n_words': 286}