<commit_before>require 'json'

Before do
  $setup ||= false
end

# Takes a path and then sequentially adds what's provided in `data`
# to be later executed in the step `the given SQL files are executed`
# +path+:: relative to root of project, e.g. "jobs/kpi/sales"
Given(/^the SQL files in the path "?(.*)"?:$/) do |path, data|
  @sql_file_path = path
  @sql_files_to_execute = data.hashes.map { |e| "#{@sql_file_path}/#{e['file']}" }
end

Given(/^the SQL file path "?([^\s]+)"?$/) do |path|
  @sql_file_path = path
end

Given(/^Pending: (.*)/) { |reason| pending(reason) }

Given(/^their schema dependencies:$/) do |data|
  unless $setup
    schemas = data.hashes.map { |hash| hash['schema'] }.compact.uniq
    TESTING_DATABASE.setup(schemas)
  end
end

Given(/^their table dependencies:$/) do |data|
  if $setup
    silence_streams(STDERR) do
      TESTING_DATABASE.truncate_all_tables()
    end
  else
    tables = []
    schemas = []
    data.hashes.each do |hash|
      schema, table = hash['table'].split('.')
      schemas << schema
      tables << { schema => table }
    end
    silence_streams(STDERR) do
      TESTING_DATABASE.setup(schemas.compact.uniq)
      TESTING_DATABASE.copy_table_defs_from_prod(tables)
    end
    $setup = true
  end
end

Given(/^the following defaults for "?([^\s]+)"? \(if not stated otherwise\):$/) do |table, data|
  @defaults ||= {}
  @defaults[table] = data.hashes[0]
end

Given(/a clean environment/) do
  silence_streams(STDERR) do
    TESTING_DATABASE.truncate_all_tables()
  end
end

Given(/^the existing table "?([a-zA-Z0-9_]+)\.([a-zA-Z0-9_]+)"?( with date placeholders)?:$/) do |schema, table, placeholder, data|
  mock_data = data.hashes
  @defaults ||= {}
  defaults = @defaults["#{schema}.#{table}"]

  unless defaults.nil? or defaults.empty?
    mock_data.map! { |entry| defaults.merge(entry) }
  end

  mock_data = convert_mock_values(mock_data) if placeholder

  TESTING_DATABASE.mock(
    Hash["#{schema}.#{table}", mock_data]
  )
end

When(/^the given SQL files are executed$/) do
  silence_streams(STDERR) do
    @sql_files_to_execute.each { |file| TESTING_DATABASE.exec_file(file) }
  end
end

When(/^the SQL file "?([^\s]+)"? is executed/) do |file|
  silence_streams(STDERR) do
    TESTING_DATABASE.exec_file("#{@sql_file_path}/#{file}")
  end
end

When(/^the resulting table "?([^\s]*)"? is queried(?:, ordered by "?(.*)"?)?/) do |table, sort_column|
  sort_statement = (sort_column.nil? or sort_column.empty?) ? '' : "order by #{sort_column}"
  @result = TESTING_DATABASE.query("select * from #{table} #{sort_statement};").map { |e| e }
end

Then(/^the result starts with.*$/) do |data|
  actual = @result[0..(data.hashes.length - 1)] || []
  expected = data.hashes || []

  expected.each_with_index do |hash, i|
    raise("Does not start with expected result, got:\n#{format_error(data, actual)}") unless actual[i].all? do |key, value|
      values_match(value, hash[key]) # actual,expected
    end
  end
end<commit_msg>Fix regex<commit_after>require 'json'

Before do
  $setup ||= false
end

# Takes a path and then sequentially adds what's provided in `data`
# to be later executed in the step `the given SQL files are executed`
# +path+:: relative to root of project, e.g. "jobs/kpi/sales"
Given(/^the SQL files in the path "?([^"]*)"?:$/) do |path, data|
  @sql_file_path = path
  @sql_files_to_execute = data.hashes.map { |e| "#{@sql_file_path}/#{e['file']}" }
end

Given(/^the SQL file path "?([^\s"]+)"?$/) do |path|
  @sql_file_path = path
end

Given(/^Pending: (.*)/) { |reason| pending(reason) }

Given(/^their schema dependencies:$/) do |data|
  unless $setup
    schemas = data.hashes.map { |hash| hash['schema'] }.compact.uniq
    TESTING_DATABASE.setup(schemas)
  end
end

Given(/^their table dependencies:$/) do |data|
  if $setup
    silence_streams(STDERR) do
      TESTING_DATABASE.truncate_all_tables()
    end
  else
    tables = []
    schemas = []
    data.hashes.each do |hash|
      schema, table = hash['table'].split('.')
      schemas << schema
      tables << { schema => table }
    end
    silence_streams(STDERR) do
      TESTING_DATABASE.setup(schemas.compact.uniq)
      TESTING_DATABASE.copy_table_defs_from_prod(tables)
    end
    $setup = true
  end
end

Given(/^the following defaults for "?([^\s"]+)"? \(if not stated otherwise\):$/) do |table, data|
  @defaults ||= {}
  @defaults[table] = data.hashes[0]
end

Given(/a clean environment/) do
  silence_streams(STDERR) do
    TESTING_DATABASE.truncate_all_tables()
  end
end

Given(/^the existing table "?([a-zA-Z0-9_]+)\.([a-zA-Z0-9_]+)"?( with date placeholders)?:$/) do |schema, table, placeholder, data|
  mock_data = data.hashes
  @defaults ||= {}
  defaults = @defaults["#{schema}.#{table}"]

  unless defaults.nil? or defaults.empty?
    mock_data.map! { |entry| defaults.merge(entry) }
  end

  mock_data = convert_mock_values(mock_data) if placeholder

  TESTING_DATABASE.mock(
    Hash["#{schema}.#{table}", mock_data]
  )
end

When(/^the given SQL files are executed$/) do
  silence_streams(STDERR) do
    @sql_files_to_execute.each { |file| TESTING_DATABASE.exec_file(file) }
  end
end

When(/^the SQL file "?([^\s]+)"? is executed/) do |file|
  silence_streams(STDERR) do
    TESTING_DATABASE.exec_file("#{@sql_file_path}/#{file}")
  end
end

When(/^the resulting table "?([^\s"]*)"? is queried(?:, ordered by "?([^"]*)"?)?/) do |table, sort_column|
  sort_statement = (sort_column.nil? or sort_column.empty?) ? '' : "order by #{sort_column}"
  @result = TESTING_DATABASE.query("select * from #{table} #{sort_statement};").map { |e| e }
end

Then(/^the result starts with.*$/) do |data|
  actual = @result[0..(data.hashes.length - 1)] || []
  expected = data.hashes || []

  expected.each_with_index do |hash, i|
    raise("Does not start with expected result, got:\n#{format_error(data, actual)}") unless actual[i].all? do |key, value|
      values_match(value, hash[key]) # actual,expected
    end
  end
end
{'fraction_non_alphanumeric': 0.17115609837711226, 'fraction_numerical': 0.0023423121967542245, 'mean_word_length': 3.973377703826955, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2271674', 'n_tokens_mistral': 2194, 'n_tokens_neox': 2147, 'n_words': 526}