<commit_before>#bulk extract
TRIGGER_SCRIPT = File.expand_path(PropLoader.getProps['bulk_extract_script'])
OUTPUT_DIRECTORY = PropLoader.getProps['bulk_extract_output_directory']
PROPERTIES_FILE = PropLoader.getProps['bulk_extract_properties_file']
KEYSTORE_FILE = PropLoader.getProps['bulk_extract_keystore_file']
JAR_FILE = PropLoader.getProps['bulk_extract_jar_loc']
SSH_USER = PropLoader.getProps['ssh_user']
EXTRACT_TO_DIRECTORY = PropLoader.getProps['extract_to_directory']

require 'archive/tar/minitar'
include Archive::Tar

Given /^the extraction zone is empty$/ do
    if (Dir.exists?(OUTPUT_DIRECTORY))
      puts OUTPUT_DIRECTORY
      FileUtils.rm_rf("#{OUTPUT_DIRECTORY}/.", secure: true)
    end
end

Given /^the production extraction zone is empty$/ do
   executeShellCommand("rm -f \~/.ssh/known_hosts")
   executeShellCommand("ssh #{SSH_USER} sudo rm -rf #{OUTPUT_DIRECTORY}#{convertTenantIdToDbName(PropLoader.getProps['tenant'])}")
end

Given /^there is no bulk extract files in the local directory$/ do
    executeShellCommand("rm -f #{EXTRACT_TO_DIRECTORY}/extract.tar")
    executeShellCommand("rm -rf #{EXTRACT_TO_DIRECTORY}/unpack/")
    executeShellCommand("rm -rf #{EXTRACT_TO_DIRECTORY}/extract/")
    executeShellCommand("rm -rf #{EXTRACT_TO_DIRECTORY}/logs/")
end

def executeShellCommand(command)
    puts command
    `#{command}`
end

When /^the operator triggers a bulk extract for the production tenant$/ do
    executeShellCommand("rm -f \~/.ssh/known_hosts")
    command = getBulkExtractCommand(PropLoader.getProps['tenant'])
    executeShellCommand("ssh #{SSH_USER} sudo #{command}")
end

When /^the operator triggers a delta for the production tenant$/ do
    executeShellCommand("rm -f \~/.ssh/known_hosts")
    command = getBulkExtractCommand(PropLoader.getProps['tenant'], " -d")
    executeShellCommand("ssh #{SSH_USER} sudo #{command}")
end

When /^I store the URL for the latest delta for the LEA$/ do
  puts "result body from previous API call is #{@res}"
  @delta_uri = JSON.parse(@res)
  @list_url  = @delta_uri["deltaLeas"][@lea][0]["uri"]
  # @list_irl is in the format https://<url>/api/rest/v1.2/bulk/extract/<lea>/delta/<timestamp>
  # -> strip off everything before v1.2, store: bulk/extract/<lea>/delta/<timestamp>
  @list_url.match(/api\/rest\/v(.*?)\/(.*)$/)
  puts "Bulk Extract Delta URI suffix: #{$2}"
  @list_uri = $2
  # Get the timestamp from the URL
  @list_url.match(/delta\/(.*)$/)
  @delta_file = "delta_#{lea}_#{$1}.tar"<commit_msg>All e2e and no play make homer.. something something<commit_after>#bulk extract
TRIGGER_SCRIPT = File.expand_path(PropLoader.getProps['bulk_extract_script'])
OUTPUT_DIRECTORY = PropLoader.getProps['bulk_extract_output_directory']
PROPERTIES_FILE = PropLoader.getProps['bulk_extract_properties_file']
KEYSTORE_FILE = PropLoader.getProps['bulk_extract_keystore_file']
JAR_FILE = PropLoader.getProps['bulk_extract_jar_loc']
SSH_USER = PropLoader.getProps['ssh_user']
EXTRACT_TO_DIRECTORY = PropLoader.getProps['extract_to_directory']

require 'archive/tar/minitar'
include Archive::Tar

Given /^the extraction zone is empty$/ do
    if (Dir.exists?(OUTPUT_DIRECTORY))
      puts OUTPUT_DIRECTORY
      FileUtils.rm_rf("#{OUTPUT_DIRECTORY}/.", secure: true)
    end
end

Given /^the production extraction zone is empty$/ do
   executeShellCommand("rm -f \~/.ssh/known_hosts")
   executeShellCommand("ssh #{SSH_USER} sudo rm -rf #{OUTPUT_DIRECTORY}#{convertTenantIdToDbName(PropLoader.getProps['tenant'])}")
end

Given /^there is no bulk extract files in the local directory$/ do
    executeShellCommand("rm -f #{EXTRACT_TO_DIRECTORY}/extract.tar")
    executeShellCommand("rm -rf #{EXTRACT_TO_DIRECTORY}/unpack/")
    executeShellCommand("rm -rf #{EXTRACT_TO_DIRECTORY}/extract/")
    executeShellCommand("rm -rf #{EXTRACT_TO_DIRECTORY}/logs/")
end

def executeShellCommand(command)
    puts command
    `#{command}`
end

When /^the operator triggers a bulk extract for the production tenant$/ do
    executeShellCommand("rm -f \~/.ssh/known_hosts")
    command = getBulkExtractCommand(PropLoader.getProps['tenant'])
    executeShellCommand("ssh #{SSH_USER} sudo #{command}")
end

When /^the operator triggers a delta for the production tenant$/ do
    executeShellCommand("rm -f \~/.ssh/known_hosts")
    command = getBulkExtractCommand(PropLoader.getProps['tenant'], " -d")
    executeShellCommand("ssh #{SSH_USER} sudo #{command}")
end

When /^I store the URL for the latest delta for the LEA$/ do
  puts "result body from previous API call is #{@res}"
  @delta_uri = JSON.parse(@res)
  @list_url  = @delta_uri["deltaLeas"][@lea][0]["uri"]
  # @list_irl is in the format https://<url>/api/rest/v1.2/bulk/extract/<lea>/delta/<timestamp>
  # -> strip off everything before v1.2, store: bulk/extract/<lea>/delta/<timestamp>
  @list_url.match(/api\/rest\/v(.*?)\/(.*)$/)
  puts "Bulk Extract Delta URI suffix: #{$2}"
  @list_uri = $2
  # Get the timestamp from the URL
  @list_url.match(/delta\/(.*)$/)
  @delta_file = "delta_#{@lea}_#{$1}.tar"
{'fraction_non_alphanumeric': 0.14152441445017863, 'fraction_numerical': 0.003374354902739182, 'mean_word_length': 5.167686658506732, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '21575', 'n_tokens_mistral': 1716, 'n_tokens_neox': 1679, 'n_words': 381}