<commit_before>require_relative '..\\bookmaker\\header.rb'

filename_split_nospaces = Bkmkr::Project.filename_split.gsub(/ /, "")

# For TEST purposes
test_images_before = Dir.entries("#{Bkmkr::Project.working_dir}\\submitted_images\\")

# Rename and move input files to tmp folder to eliminate possibility of overwriting
`md #{tmp_dir}\\#{Bkmkr::Project.filename}`
`md #{tmp_dir}\\#{Bkmkr::Project.filename}\\images`
`move #{Bkmkr::Project.working_dir}\\submitted_images\\* #{tmp_dir}\\#{Bkmkr::Project.filename}\\images\\`
`copy "#{Bkmkr::Project.input_file}" #{tmp_dir}\\#{Bkmkr::Project.filename}\\#{filename_split_nospaces}`

# Add a notice to the conversion dir warning that the process is in use
File.open("#{Bkmkr::Project.working_dir}\\IN_USE_PLEASE_WAIT.txt", 'w') do |output|
	output.write "The conversion processor is currently running. Please do not submit any new files or images until the process completes."
end

# TESTING

# Filename should not be null
test_fname = Bkmkr::Project.filename.split(%r{\s*})

if test_fname.length != 0
	test_fname_status = "pass: original filename is not null"
else
	test_fname_status = "FAIL: original filename is not null"
end

# tmpdir should exist
if File.exist?("#{tmp_dir}\\#{Bkmkr::Project.filename}") and File.exist?("#{tmp_dir}\\#{Bkmkr::Project.filename}\\images")
	test_dir_status = "pass: temp directory and all sub-directories were successfully created"
else
	test_dir_status = "FAIL: temp directory and all sub-directories were successfully created"
end

# submitted images dir should be clean
test_images_after = Dir.entries("#{Bkmkr::Project.working_dir}\\submitted_images\\")

if test_images_after.length == 2
	test_imagedir_status = "pass: submitted images directory has been emptied"
else
	test_imagedir_status = "FAIL: submitted images directory has been emptied"
end

# IF submitted images dir was not clean at beginning, tmpdir images dir should also not be clean at end
test_tmp_images = Dir.entries("#{tmp_dir}\\#{Bkmkr::Project.filename}\\images\\")

if test_images_before.length == test_tmp_images.length
	test_tmpimgdir_status = "pass: all submitted images have been copied to temp directory"
else
	test_tmpimgdir_status = "FAIL: all submitted images have been copied to temp directory"
end

# input file should exist in tmp dir
if File.file?("#{tmp_dir}\\#{Bkmkr::Project.filename}\\#{filename_split_nospaces}")
	test_input_status = "pass: original file preserved in project directory"
else
	test_input_status = "FAIL: original file preserved in project directory"
end

# Write test results
File.open("#{Bkmkr::Dir.log_dir}\\#{Bkmkr::Project.filename}.txt", 'a+') do |f|
	f.puts "-----"
	f.puts Time.now
	f.puts "----- TMPARCHIVE PROCESSES"
	f.puts test_fname_status
	f.puts test_dir_status
	f.puts test_imagedir_status
	f.puts test_tmpimgdir_status
	f.puts test_input_status
end<commit_msg>updating header<commit_after>require_relative '..\\bookmaker\\header.rb'

filename_split_nospaces = Bkmkr::Project.filename_split.gsub(/ /, "")

# For TEST purposes
test_images_before = Dir.entries("#{Bkmkr::Project.working_dir}\\submitted_images\\")

# Rename and move input files to tmp folder to eliminate possibility of overwriting
`md #{Bkmkr::Dir.tmp_dir}\\#{Bkmkr::Project.filename}`
`md #{Bkmkr::Dir.tmp_dir}\\#{Bkmkr::Project.filename}\\images`
`move #{Bkmkr::Project.working_dir}\\submitted_images\\* #{Bkmkr::Dir.tmp_dir}\\#{Bkmkr::Project.filename}\\images\\`
`copy "#{Bkmkr::Project.input_file}" #{Bkmkr::Dir.tmp_dir}\\#{Bkmkr::Project.filename}\\#{filename_split_nospaces}`

# Add a notice to the conversion dir warning that the process is in use
File.open("#{Bkmkr::Project.working_dir}\\IN_USE_PLEASE_WAIT.txt", 'w') do |output|
	output.write "The conversion processor is currently running. Please do not submit any new files or images until the process completes."
end

# TESTING

# Filename should not be null
test_fname = Bkmkr::Project.filename.split(%r{\s*})

if test_fname.length != 0
	test_fname_status = "pass: original filename is not null"
else
	test_fname_status = "FAIL: original filename is not null"
end

# tmpdir should exist
if File.exist?("#{Bkmkr::Dir.tmp_dir}\\#{Bkmkr::Project.filename}") and File.exist?("#{Bkmkr::Dir.tmp_dir}\\#{Bkmkr::Project.filename}\\images")
	test_dir_status = "pass: temp directory and all sub-directories were successfully created"
else
	test_dir_status = "FAIL: temp directory and all sub-directories were successfully created"
end

# submitted images dir should be clean
test_images_after = Dir.entries("#{Bkmkr::Project.working_dir}\\submitted_images\\")

if test_images_after.length == 2
	test_imagedir_status = "pass: submitted images directory has been emptied"
else
	test_imagedir_status = "FAIL: submitted images directory has been emptied"
end

# IF submitted images dir was not clean at beginning, tmpdir images dir should also not be clean at end
test_tmp_images = Dir.entries("#{Bkmkr::Dir.tmp_dir}\\#{Bkmkr::Project.filename}\\images\\")

if test_images_before.length == test_tmp_images.length
	test_tmpimgdir_status = "pass: all submitted images have been copied to temp directory"
else
	test_tmpimgdir_status = "FAIL: all submitted images have been copied to temp directory"
end

# input file should exist in tmp dir
if File.file?("#{Bkmkr::Dir.tmp_dir}\\#{Bkmkr::Project.filename}\\#{filename_split_nospaces}")
	test_input_status = "pass: original file preserved in project directory"
else
	test_input_status = "FAIL: original file preserved in project directory"
end

# Write test results
File.open("#{Bkmkr::Dir.log_dir}\\#{Bkmkr::Project.filename}.txt", 'a+') do |f|
	f.puts "-----"
	f.puts Time.now
	f.puts "----- TMPARCHIVE PROCESSES"
	f.puts test_fname_status
	f.puts test_dir_status
	f.puts test_imagedir_status
	f.puts test_tmpimgdir_status
	f.puts test_input_status
end
{'fraction_non_alphanumeric': 0.12950875987633115, 'fraction_numerical': 0.0006870491240123669, 'mean_word_length': 5.632118451025057, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '53704', 'n_tokens_mistral': 1877, 'n_tokens_neox': 1896, 'n_words': 510}