<commit_before>ENV['DATA_MODEL'] ||= 'omopv4'

require_relative 'db'

if ENV['COVERAGE']
  require 'coverage'
  require 'simplecov'

  ENV.delete('COVERAGE')
  SimpleCov.instance_exec do
    start do
      add_filter "/test/"
      add_group('Missing'){|src| src.covered_percent < 100}
      add_group('Covered'){|src| src.covered_percent == 100}
      yield self if block_given?
    end
  end
end

$: << "lib"
require 'conceptql'
require 'minitest/spec'
require 'minitest/autorun'

require 'logger'
require 'pp'
require 'fileutils'

CDB = ConceptQL::Database.new(DB, :data_model=>ENV['DATA_MODEL'].to_sym)
DB.extension :error_sql

#ENV["OVERWRITE_CONCEPTQL_TEST_RESULTS"] = '1'

class Minitest::Spec
  def annotate(test_name, statement=nil)
    load_check(test_name, statement){|statement| query(statement).annotate}
  end

  def scope_annotate(test_name, statement=nil)
    load_check(test_name, statement){|statement| query(statement).scope_annotate}
  end

  def domains(test_name, statement=nil)
    load_check(test_name, statement){|statement| query(statement).domains}
  end

  def query(statement)
    CDB.query(statement)
  end

  def dataset(statement)
    statement = query(statement) unless statement.is_a?(ConceptQL::Query)
    statement.query
  end

  def criteria_ids(test_name, statement=nil)
    load_check(test_name, statement){|statement| hash_groups(statement, :criterion_domain, :criterion_id)}
  end

  # If no statement is passed, this function loads the statement from the specified test
  # file. If a statement is passed, it is written to the file.
  def load_statement(test_name, statement)
    path = "test/statements/#{test_name}"
    if statement
      FileUtils.mkdir_p(File.dirname(path))
      File.write(path, JSON.pretty_generate(statement))
      statement
    else
      JSON.parse(File.read(path))
    end
  end

  def check_output(test_name, results)
    path = "test/results/#{ENV["DATA_MODEL"]}/#{test_name}"
    if ENV["OVERWRITE_CONCEPTQL_TEST_RESULTS"]
      json = JSON.pretty_generate(results)
      FileUtils.mkdir_p(File.dirname(path))
      File.write(path, JSON.pretty_generate(results))
    else
      JSON.parse(results.to_json).must_equal(JSON.parse(File.read(path)))
    end
    results
  end

  def numeric_values(test_name, statement=nil)
    load_check(test_name, statement){|statement| hash_groups(statement, :criterion_domain, :value_as_number)}
  end

  def criteria_counts(test_name, statement=nil)
    load_check(test_name, statement){|statement| query(statement).query.from_self.group_and_count(:criterion_domain).to_hash(:criterion_domain, :count)}
  end

  def optimized_criteria_counts(test_name, statement=nil)
    load_check(test_name, statement){|statement| query(statement).optimized.query.from_self.group_and_count(:criterion_domain).to_hash(:criterion_domain, :count)}
  end

  def hash_groups(statement, key, value)
    dataset(statement).from_self.distinct.order(*value).to_hash_groups(key, value)
  rescue
    puts $!.sql if $!.respond_to?(:sql)
    raise
  end
<commit_msg>Test statements with no expected results now expect []<commit_after>ENV['DATA_MODEL'] ||= 'omopv4'

require_relative 'db'

if ENV['COVERAGE']
  require 'coverage'
  require 'simplecov'

  ENV.delete('COVERAGE')
  SimpleCov.instance_exec do
    start do
      add_filter "/test/"
      add_group('Missing'){|src| src.covered_percent < 100}
      add_group('Covered'){|src| src.covered_percent == 100}
      yield self if block_given?
    end
  end
end

$: << "lib"
require 'conceptql'
require 'minitest/spec'
require 'minitest/autorun'

require 'logger'
require 'pp'
require 'fileutils'

CDB = ConceptQL::Database.new(DB, :data_model=>ENV['DATA_MODEL'].to_sym)
DB.extension :error_sql

#ENV["OVERWRITE_CONCEPTQL_TEST_RESULTS"] = '1'

class Minitest::Spec
  def annotate(test_name, statement=nil)
    load_check(test_name, statement){|statement| query(statement).annotate}
  end

  def scope_annotate(test_name, statement=nil)
    load_check(test_name, statement){|statement| query(statement).scope_annotate}
  end

  def domains(test_name, statement=nil)
    load_check(test_name, statement){|statement| query(statement).domains}
  end

  def query(statement)
    CDB.query(statement)
  end

  def dataset(statement)
    statement = query(statement) unless statement.is_a?(ConceptQL::Query)
    statement.query
  end

  def criteria_ids(test_name, statement=nil)
    load_check(test_name, statement){|statement| hash_groups(statement, :criterion_domain, :criterion_id)}
  end

  # If no statement is passed, this function loads the statement from the specified test
  # file. If a statement is passed, it is written to the file.
  def load_statement(test_name, statement)
    path = "test/statements/#{test_name}"
    if statement
      FileUtils.mkdir_p(File.dirname(path))
      File.write(path, JSON.pretty_generate(statement))
      statement
    else
      JSON.parse(File.read(path))
    end
  end

  def check_output(test_name, results)
    path = "test/results/#{ENV["DATA_MODEL"]}/#{test_name}"

    if ENV["OVERWRITE_CONCEPTQL_TEST_RESULTS"]
      save_results(path, results)
    end

    expected = begin
      File.read(path)
    rescue Errno::ENOENT
      save_results(path, [])
      "[]"
    end

    JSON.parse(results.to_json).must_equal(JSON.parse(expected))
    results
  end

  def save_results(path, results)
    FileUtils.mkdir_p(File.dirname(path))
    File.write(path, JSON.pretty_generate(results))
  end

  def numeric_values(test_name, statement=nil)
    load_check(test_name, statement){|statement| hash_groups(statement, :criterion_domain, :value_as_number)}
  end

  def criteria_counts(test_name, statement=nil)
    load_check(test_name, statement){|statement| query(statement).query.from_self.group_and_count(:criterion_domain).to_hash(:criterion_domain, :count)}
  end

  def optimized_criteria_counts(test_name, statement=nil)
    load_check(test_name, statement){|statement| query(statement).optimized.query.from_self.group_and_count(:criterion_domain).to_hash(:criterion_domain, :count)}
  end

  def hash_groups(statement, key, value)
    dataset(statement).from_self.distinct.order(*value).to_hash_groups(key, value)
  rescue
    puts $!.sql if $!.respond_to?(:sql)
    raise
  end

{'fraction_non_alphanumeric': 0.11742182514358647, 'fraction_numerical': 0.0025526483726866626, 'mean_word_length': 4.880863039399625, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3165882', 'n_tokens_mistral': 2132, 'n_tokens_neox': 2061, 'n_words': 398}