<commit_before>require 'test_helper'
require File.dirname(__FILE__) + "/../vendor/treetop/lib/treetop"
Treetop.load "../lib/parser/personify"
# require File.dirname(__FILE__) + "/../personify"

class ParserTest < Test::Unit::TestCase
  include ParserTestHelper
  context "The parser" do
    setup do
      @parser = PersonifyLanguageParser.new
    end
    
    context "parsing keys" do
      should "eval [A] as key" do
        assert_equal "test", parse("[A]").eval({"a" => "test"})
      end
      
      should "eval [A_B] as key" do
        assert_equal "test", parse("[A_B]").eval({"a_b" => "test"})        
      end      
      
      should "eval [A.B] as nested key" do
        assert_equal "test", parse("[A.B]").eval({"a.b" => "fail", "a" => {"b" => "test"}})
      end
            
      should "eval [AB_C.D] as nested key" do
        assert_equal "test", parse("[AB_C.D]").eval({"ab_c.d" => "fail", "ab_c" => {"d" => "test"}})
      end
            
      should "eval [L1.L3.L4.L5.L6.L7.L8] as nested key" do
        assert_equal "test", parse("[L1.L3.L4.L5.L6.L7.L8]").eval({"l1" => {"l2" => {"l3" => {"l4" => {"l5" => {"l6" => {"l7" => {"l8" => "test"}}}}}}}})          
      end
      
      should "eval [1.2] as nested key" do
        assert_equal "test", parse("[1.2]").eval({1 => {2 => "test"}})
      end
    end
    context "parsing text" do
      should "eval text" do
        assert_equal "text", parse("text").eval()
        assert_equal "t", parse("t").eval()
        assert_equal "t\n1\n2", parse("t\n1\n2").eval()       
      end
      
      should "eval UTF8 text" do
        assert_equal "financiële", parse("financiële").eval()
      end
  
      should "eval empty text" do
        assert_equal "", parse("").eval()
      end
  
      should "eval '[bla]' as text" do
        assert_equal "[bla]", parse("[bla]").eval()
      end

      should "eval '[KE bla' as text" do
        assert_equal "[KE bla", parse("[KE bla").eval()
      end

      should "eval nested brackets as text" do
        assert_equal "[[BLA]]", parse("[[BLA]]").eval
      end

      should "eval '[BLA]' as text on nil 'bla' " do
        assert_equal "[BLA]", parse("[BLA]").eval()
      end
    end
    context "parsing expressions" do  
      should "eval simple expression" do
        assert_equal "var", parse("[VAR]").eval("var" => "var")
        assert_equal "a var b", parse("a [VAR] b").eval("var" => "var")
        assert_equal "var\nvar", parse("[VAR]\n[VAR]").eval("var" => "var")              
      end
      
      should "eval simple expression with empty substitution" do
        assert_equal "", parse("[VAR]").eval("var" => "")
      end
      
      should "eval expressions careless of whitespace" do
        assert_equal "k1",parse("[ K1]").eval("k1" => "k1")
        assert_equal "k1",parse("[ K1 ]").eval("k1" => "k1")
        assert_equal "k1",parse("[K1 | K2]").eval("k1" => "k1")
      end   
     
      context "with alternatives" do    
        should "eval alternative expression on first empty" do       
          assert_equal "k2", parse("[K1|K2]").eval("k2" => "k2")
          assert_equal "k2", parse("[K1|K2]").eval("k2" => "k2")
          assert_equal "k3", parse("[K1|K2|K3]").eval("k1" => nil,"k2" => nil, "k3" => "k3")
        end

        should "eval first expression on first nonempty" do       
          assert_equal "k2", parse("[K1|K2]").eval("k2" => "k2")
          assert_equal "k3", parse("[K1|K2|K3]").eval("k2" => nil, "k3" => "k3")
        end     

        should "eval strings in alternative expression" do <commit_msg>Add string parsing test<commit_after>require 'test_helper'
require File.dirname(__FILE__) + "/../vendor/treetop/lib/treetop"
Treetop.load "../lib/parser/personify"
# require File.dirname(__FILE__) + "/../personify"

class ParserTest < Test::Unit::TestCase
  include ParserTestHelper
  context "The parser" do
    setup do
      @parser = PersonifyLanguageParser.new
    end
    
    context "parsing keys" do
      should "eval [A] as key" do
        assert_equal "test", parse("[A]").eval({"a" => "test"})
      end
      
      should "eval [A_B] as key" do
        assert_equal "test", parse("[A_B]").eval({"a_b" => "test"})        
      end      
      
      should "eval [A.B] as nested key" do
        assert_equal "test", parse("[A.B]").eval({"a.b" => "fail", "a" => {"b" => "test"}})
      end
            
      should "eval [AB_C.D] as nested key" do
        assert_equal "test", parse("[AB_C.D]").eval({"ab_c.d" => "fail", "ab_c" => {"d" => "test"}})
      end
            
      should "eval [L1.L3.L4.L5.L6.L7.L8] as nested key" do
        assert_equal "test", parse("[L1.L3.L4.L5.L6.L7.L8]").eval({"l1" => {"l2" => {"l3" => {"l4" => {"l5" => {"l6" => {"l7" => {"l8" => "test"}}}}}}}})          
      end
      
      should "eval [1.2] as nested key" do
        assert_equal "test", parse("[1.2]").eval({1 => {2 => "test"}})
      end
    end
    
    context "parsing strings" do
      should "eval [\"str\"] as string" do
        assert_equal "str", parse("[\"str\"]").eval({})
      end
      should "eval [str] as string" do
        assert_equal "str", parse("[str]").eval({})
      end
    end
    
    context "parsing text" do
      should "eval text" do
        assert_equal "text", parse("text").eval()
        assert_equal "t", parse("t").eval()
        assert_equal "t\n1\n2", parse("t\n1\n2").eval()       
      end
      
      should "eval UTF8 text" do
        assert_equal "financiële", parse("financiële").eval()
      end
  
      should "eval empty text" do
        assert_equal "", parse("").eval()
      end
  
      should "eval '[bla]' as text" do
        assert_equal "[bla]", parse("[bla]").eval()
      end

      should "eval '[KE bla' as text" do
        assert_equal "[KE bla", parse("[KE bla").eval()
      end

      should "eval nested brackets as text" do
        assert_equal "[[BLA]]", parse("[[BLA]]").eval
      end

      should "eval '[BLA]' as text on nil 'bla' " do
        assert_equal "[BLA]", parse("[BLA]").eval()
      end
    end
    
    context "parsing expressions" do  
      should "eval simple expression" do
        assert_equal "var", parse("[VAR]").eval("var" => "var")
        assert_equal "a var b", parse("a [VAR] b").eval("var" => "var")
        assert_equal "var\nvar", parse("[VAR]\n[VAR]").eval("var" => "var")              
      end
      
      should "eval simple expression with empty substitution" do
        assert_equal "", parse("[VAR]").eval("var" => "")
      end
      
      should "eval expressions careless of whitespace" do
        assert_equal "k1",parse("[ K1]").eval("k1" => "k1")
        assert_equal "k1",parse("[ K1 ]").eval("k1" => "k1")
        assert_equal "k1",parse("[K1 | K2]").eval("k1" => "k1")
      end   
     
      context "with alternatives" do    
        should "eval alternative expression on first empty" do       
          assert_equal "k2", parse("[K1|K2]").eval("k2" => "k2")
          assert_equal "k2", parse("[K1|K2]").eval("k2" => "k2")
          assert_equal "k3", parse("[K1|K2|K3]").eval("k1" => nil,"k2" => nil, "k3" => "k3")
        end

        should "eval first expression on first nonempty" do       
          assert_equal "k2", parse("[K1|K2]").eval("k2" => "k2")
          assert_equal "k3", parse("[K1|K2|K3]").eval("k2" => nil, "k3" => "k3")
        end     

        should "eval strings in alternative expression" do 
{'fraction_non_alphanumeric': 0.18904403866809882, 'fraction_numerical': 0.02040816326530612, 'mean_word_length': 2.2699736611062336, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2839463', 'n_tokens_mistral': 2576, 'n_tokens_neox': 2580, 'n_words': 644}