<commit_before>$:.unshift(File.dirname(File.dirname(File.expand_path(__FILE__))))
$:.unshift(File.dirname(File.expand_path(__FILE__)))

require 'automata_test'
require 'test/unit'

if ! ENV['abs_builddir']
  raise "Need environmental variable abs_builddir properly set."
end

def random_word(max_length)
  length = rand(max_length) + 1
  (1..length).collect {"%c" % (97 + rand(26))}.join
end

class TestBasic < Test::Unit::TestCase
  include AutomataTest

  def test_traditional
    words = ["he", "she", "his", "hers"]
    text = "she saw his world as he saw hers..."

    ac_test(words, text, "traditional")
    ac_test(words, text, "traditional_optimized", true)
  end

  def test_large
    n = 1000

    words = Set.new
    while words.size < n
      words << random_word(10)
    end
    words = words.to_a

    text = words.join(" ")

    ac_test(words, text, "large")
  end

  def test_moderate
    n = 200

    words = Set.new
    while words.size < n
      words << random_word(10)
    end
    words = words.to_a

    text = words.join(" ")

    ac_test(words, text, "moderate")
    ac_test(words, text, "moderate_optimized", true)
  end

  def test_aaaa
    words = ["a", "aa", "aaa", "aaaa"]
    text = "aaaaaaaaaaaa"

    ac_test(words, text, "aaaa")
    ac_test(words, text, "aaaa_optimized", true)
  end

  def test_wide
    words = []
    ('a'..'z').each do |x|
      words << "a#{x}"
      words << "a#{x.upcase}"
    end
    text = words.join(" ")

    ac_test(words, text, "wide")<commit_msg>automata/testing: Expand wide test to better catch issues.<commit_after>$:.unshift(File.dirname(File.dirname(File.expand_path(__FILE__))))
$:.unshift(File.dirname(File.expand_path(__FILE__)))

require 'automata_test'
require 'test/unit'

if ! ENV['abs_builddir']
  raise "Need environmental variable abs_builddir properly set."
end

def random_word(max_length)
  length = rand(max_length) + 1
  (1..length).collect {"%c" % (97 + rand(26))}.join
end

class TestBasic < Test::Unit::TestCase
  include AutomataTest

  def test_traditional
    words = ["he", "she", "his", "hers"]
    text = "she saw his world as he saw hers..."

    ac_test(words, text, "traditional")
    ac_test(words, text, "traditional_optimized", true)
  end

  def test_large
    n = 1000

    words = Set.new
    while words.size < n
      words << random_word(10)
    end
    words = words.to_a

    text = words.join(" ")

    ac_test(words, text, "large")
  end

  def test_moderate
    n = 200

    words = Set.new
    while words.size < n
      words << random_word(10)
    end
    words = words.to_a

    text = words.join(" ")

    ac_test(words, text, "moderate")
    ac_test(words, text, "moderate_optimized", true)
  end

  def test_aaaa
    words = ["a", "aa", "aaa", "aaaa"]
    text = "aaaaaaaaaaaa"

    ac_test(words, text, "aaaa")
    ac_test(words, text, "aaaa_optimized", true)
  end

  def test_wide
    words = []
    ('a'..'z').each do |x|
      words << "a#{x}#{x}"
      words << "a#{x.upcase}#{x}"
    end
    text = words.join(" ")

    ac_test(words, text, "wide")
{'fraction_non_alphanumeric': 0.15011396939107782, 'fraction_numerical': 0.011071312276131553, 'mean_word_length': 3.096, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4907394', 'n_tokens_mistral': 1151, 'n_tokens_neox': 1076, 'n_words': 270}