<commit_before>class TestIndexMatch < QueryOptimizerTestCase
  def setup
    Groonga::Schema.define do |schema|
      schema.create_table("Logs") do |table|
        table.text("message")
      end

      schema.create_table("Terms",
                          :type => :patricia_trie,
                          :default_tokenizer => "TokenBigram",
                          :normalizer => "NormalizerAuto") do |table|
        table.index("Logs", "message")
      end
    end

    @logs = Groonga["Logs"]
    setup_expression(@logs)
  end

  def teardown
    teardown_expression
  end

  def test_only
    assert_equal(<<-DUMP, dump_plan("message @ 'Groonga'"))
[0]
  op:         <match>
  logical_op: <or>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Groonga">
  expr:       <0..2>
    DUMP
  end

  def test_with_arithmetic_operator
    assert_equal(<<-DUMP, dump_plan("message @ 'Groonga' && ((1 + 1) == 2)"))
[0]
  op:         <match>
  logical_op: <or>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Groonga">
  expr:       <0..2>
[1]
  op:         <equal>
  logical_op: <and>
  index:      <[]>
  query:      <2>
  expr:       <3..7>
    DUMP
  end

  def test_not
    assert_equal(<<-DUMP, dump_plan("!(message @ 'Groonga')"))
[0]
  op:         <call>
  logical_op: <or>
  args[0]:    <#<proc:function all_records arguments:[]>>
  expr:       <0..0>
[1]
  op:         <match>
  logical_op: <and_not>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Groonga">
  expr:       <0..2>
    DUMP
  end

  def test_not_and_normal
    assert_equal(<<-DUMP, dump_plan("!(message @ 'Groonga') && (message @ 'Rroonga')"))
[0]
  op:         <call>
  logical_op: <or>
  args[0]:    <#<proc:function all_records arguments:[]>>
  expr:       <0..0>
[1]
  op:         <match>
  logical_op: <and_not>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Groonga">
  expr:       <0..2>
[2]
  op:         <match>
  logical_op: <and>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Rroonga">
  expr:       <4..6>
    DUMP
  end

  def test_normal_and_not
    assert_equal(<<-DUMP, dump_plan("(message @ 'Rroonga') && !(message @ 'Groonga')"))
[0]
  op:         <match>
  logical_op: <or>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Rroonga">
  expr:       <0..2>
[1]
  op:         <match>
  logical_op: <and_not>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Groonga">
  expr:       <3..5>
    DUMP
  end

  def test_normal_and_not_not
    assert_equal(<<-DUMP, dump_plan("(message @ 'Rroonga') &! !(message @ 'Groonga')"))
[0]
  op:         <match>
  logical_op: <or>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Rroonga">
  expr:       <0..2>
[1]
  op:         <match>
  logical_op: <and>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Groonga">
  expr:       <3..5>
    DUMP
  end

  def test_normal_or_not
    assert_equal(<<-DUMP, dump_plan("(message @ 'Rroonga') || !(message @ 'Groonga')"))
[0]
  op:         <call>
  logical_op: <or>
  args[0]:    <#<proc:function all_records arguments:[]>>
  expr:       <0..0>
[1]
  op:         <match>
  logical_op: <and_not>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Groonga">
  expr:       <3..5>
[2]
  op:         <match>
  logical_op: <or>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Rroonga">
  expr:       <0..2>
    DUMP
  end

  sub_test_case("optimization by estimate size") do
    def setup
      super
      10.times do |i|
        @logs.add(:message => "Groonga#{i}")
      end
      2.times do |i|
        @logs.add(:message => "Rroonga#{i}")
      end
      100.times do |i|
        @logs.add(:message => "Mroonga#{i}")
      end
    end

    def test_and
      filter = "(message @ 'Groonga') && (message @ 'Rroonga')"
      assert_equal(<<-DUMP, dump_plan(filter))
[0]
  op:         <match>
  logical_op: <or>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Rroonga">
  expr:       <3..5>
[1]
  op:         <match>
  logical_op: <and>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Groonga">
  expr:       <0..2>
      DUMP
    end

    def test_and_and_not
      filter = "(message @ 'Groonga') && "
      filter << "(message @ 'Rroonga') &! "
      filter << "(message @ 'Mroonga')"
      assert_equal(<<-DUMP, dump_plan(filter))
[0]
  op:         <match>
  logical_op: <or>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Rroonga">
  expr:       <3..5>
[1]
  op:         <match>
  logical_op: <and>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Groonga">
  expr:       <0..2>
[2]
  op:         <match>
  logical_op: <and_not>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Mroonga">
  expr:       <7..9>
      DUMP
    end
  end
end
<commit_msg>test: omit for now<commit_after>class TestIndexMatch < QueryOptimizerTestCase
  def setup
    Groonga::Schema.define do |schema|
      schema.create_table("Logs") do |table|
        table.text("message")
      end

      schema.create_table("Terms",
                          :type => :patricia_trie,
                          :default_tokenizer => "TokenBigram",
                          :normalizer => "NormalizerAuto") do |table|
        table.index("Logs", "message")
      end
    end

    @logs = Groonga["Logs"]
    setup_expression(@logs)
  end

  def teardown
    teardown_expression
  end

  def test_only
    assert_equal(<<-DUMP, dump_plan("message @ 'Groonga'"))
[0]
  op:         <match>
  logical_op: <or>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Groonga">
  expr:       <0..2>
    DUMP
  end

  def test_with_arithmetic_operator
    assert_equal(<<-DUMP, dump_plan("message @ 'Groonga' && ((1 + 1) == 2)"))
[0]
  op:         <match>
  logical_op: <or>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Groonga">
  expr:       <0..2>
[1]
  op:         <equal>
  logical_op: <and>
  index:      <[]>
  query:      <2>
  expr:       <3..7>
    DUMP
  end

  def test_not
    assert_equal(<<-DUMP, dump_plan("!(message @ 'Groonga')"))
[0]
  op:         <call>
  logical_op: <or>
  args[0]:    <#<proc:function all_records arguments:[]>>
  expr:       <0..0>
[1]
  op:         <match>
  logical_op: <and_not>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Groonga">
  expr:       <0..2>
    DUMP
  end

  def test_not_and_normal
    assert_equal(<<-DUMP, dump_plan("!(message @ 'Groonga') && (message @ 'Rroonga')"))
[0]
  op:         <call>
  logical_op: <or>
  args[0]:    <#<proc:function all_records arguments:[]>>
  expr:       <0..0>
[1]
  op:         <match>
  logical_op: <and_not>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Groonga">
  expr:       <0..2>
[2]
  op:         <match>
  logical_op: <and>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Rroonga">
  expr:       <4..6>
    DUMP
  end

  def test_normal_and_not
    assert_equal(<<-DUMP, dump_plan("(message @ 'Rroonga') && !(message @ 'Groonga')"))
[0]
  op:         <match>
  logical_op: <or>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Rroonga">
  expr:       <0..2>
[1]
  op:         <match>
  logical_op: <and_not>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Groonga">
  expr:       <3..5>
    DUMP
  end

  def test_normal_and_not_not
    assert_equal(<<-DUMP, dump_plan("(message @ 'Rroonga') &! !(message @ 'Groonga')"))
[0]
  op:         <match>
  logical_op: <or>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Rroonga">
  expr:       <0..2>
[1]
  op:         <match>
  logical_op: <and>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Groonga">
  expr:       <3..5>
    DUMP
  end

  def test_normal_or_not
    assert_equal(<<-DUMP, dump_plan("(message @ 'Rroonga') || !(message @ 'Groonga')"))
[0]
  op:         <call>
  logical_op: <or>
  args[0]:    <#<proc:function all_records arguments:[]>>
  expr:       <0..0>
[1]
  op:         <match>
  logical_op: <and_not>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Groonga">
  expr:       <3..5>
[2]
  op:         <match>
  logical_op: <or>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Rroonga">
  expr:       <0..2>
    DUMP
  end

  sub_test_case("optimization by estimate size") do
    def setup
      super
      10.times do |i|
        @logs.add(:message => "Groonga#{i}")
      end
      2.times do |i|
        @logs.add(:message => "Rroonga#{i}")
      end
      100.times do |i|
        @logs.add(:message => "Mroonga#{i}")
      end
    end

    def test_and
      omit("AND only expression is processed in C for now. " +
           "We should reconsider about it. (Optimize in C or mruby?)")
      filter = "(message @ 'Groonga') && (message @ 'Rroonga')"
      assert_equal(<<-DUMP, dump_plan(filter))
[0]
  op:         <match>
  logical_op: <or>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Rroonga">
  expr:       <3..5>
[1]
  op:         <match>
  logical_op: <and>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Groonga">
  expr:       <0..2>
      DUMP
    end

    def test_and_and_not
      filter = "(message @ 'Groonga') && "
      filter << "(message @ 'Rroonga') &! "
      filter << "(message @ 'Mroonga')"
      assert_equal(<<-DUMP, dump_plan(filter))
[0]
  op:         <match>
  logical_op: <or>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Rroonga">
  expr:       <3..5>
[1]
  op:         <match>
  logical_op: <and>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Groonga">
  expr:       <0..2>
[2]
  op:         <match>
  logical_op: <and_not>
  index:      <[#<column:index Terms.Logs_message range:Logs sources:[Logs.message] flags:POSITION>]>
  query:      <"Mroonga">
  expr:       <7..9>
      DUMP
    end
  end
end

{'fraction_non_alphanumeric': 0.16564734257903352, 'fraction_numerical': 0.012576449306572486, 'mean_word_length': 2.5723076923076924, 'pattern_counts': {'":': 0, '<': 281, '<?xml version=': 0, '>': 247, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3744693', 'n_tokens_mistral': 4602, 'n_tokens_neox': 4495, 'n_words': 886}