<commit_before>local tomoauto_directory = os.getenv('TOMOAUTOROOT')
package.cpath = package.cpath .. ';' .. tomoauto_directory .. '/lib/?.so;'
package.path  = package.path  .. ';' ..tomoauto_directory .. '/lib/?.lua;'
local MRC_IO_lib = require 'MRC_IO_lib'
local lfs        = require 'lfs'

local tomoauto_lib = {}

--[[===========================================================================#
#                                   clean_up                                   #
#------------------------------------------------------------------------------#
# This is a function that removes all the generated intermediate files in case #
# tomoauto fails to finish correctly, or when it finishes.                     #
#------------------------------------------------------------------------------#
# Arguments: basename: Image stack filename base <string>                      #
#===========================================================================--]]
function tomoauto_lib.clean_up(basename)
   local image_stack_filename           = basename .. '.st'
   local raw_tilt_filename              = basename .. '.rawtlt'
   local ccd_erased_filename            = basename .. '_fixed.st'
   local peak_model_filename            = basename .. '_peak.mod'
   local original_filename              = basename .. '_orig.st'
   local pre_xf_filename                = basename .. '.prexf'
   local pre_xg_filename                = basename .. '.prexg'
   local pre_aligned_filename           = basename .. '.preali'
   local fiducial_model_fixed_filename  = basename .. '_beadtrack.fid'
   local three_d_model_filename         = basename .. '.3dmod'
   local fiducial_xyz_filename          = basename .. 'fid.xyz'
   local residual_model_filename        = basename .. '.resid'
   local fiducial_text_model_filename   = basename .. '.fid.txt'
   local fiducial_xf_filename           = basename .. '_fid.xf'
   local xf_filename                    = basename .. '.xf'
   local fiducial_tilt_filename         = basename .. '_fid.tlt'
   local defocus_filename               = basename .. '.defocus'
   local first_aligned_filename         = basename .. '_first.ali'
   local ctf_corrected_aligned_filename = basename .. '_ctfcorr.ali'
   local xtilt_filename                 = basename .. '.xtilt'
   local gold_erase_model_filename      = basename .. '_erase.fid'
   local gold_erase_filename            = basename .. '_erase.ali'
   local second_aligned_filename        = basename .. '_second.ali'
   local RAPTOR_directory_name          = basename .. '_RAPTOR'
   local com_filenames                  = basename .. '_*.com'
   local log_filenames                  = basename .. '_*.log'
   local temporary_filenames            = basename .. '*~'

   pcall(os.execute, string.format(
         'rm -rf' .. string.rep(' %s ', 25),
         image_stack_filename,
         raw_tilt_filename,
         ccd_erased_filename,
         peak_model_filename,
         pre_xf_filename,
         pre_xg_filename,
         pre_aligned_filename,
         fiducial_model_fixed_filename,
         three_d_model_filename,
         fiducial_xyz_filename,
         residual_model_filename,
         fiducial_text_model_filename,
         fiducial_xf_filename,
         xf_filename,
         fiducial_tilt_filename,<commit_msg>Huge error with stacks getting erased!<commit_after>local tomoauto_directory = os.getenv('TOMOAUTOROOT')
package.cpath = package.cpath .. ';' .. tomoauto_directory .. '/lib/?.so;'
package.path  = package.path  .. ';' ..tomoauto_directory .. '/lib/?.lua;'
local MRC_IO_lib = require 'MRC_IO_lib'
local lfs        = require 'lfs'

local tomoauto_lib = {}

--[[===========================================================================#
#                                   clean_up                                   #
#------------------------------------------------------------------------------#
# This is a function that removes all the generated intermediate files in case #
# tomoauto fails to finish correctly, or when it finishes.                     #
#------------------------------------------------------------------------------#
# Arguments: basename: Image stack filename base <string>                      #
#===========================================================================--]]
function tomoauto_lib.clean_up(basename)
   local image_stack_filename           = basename .. '.st'
   local raw_tilt_filename              = basename .. '.rawtlt'
   local ccd_erased_filename            = basename .. '_fixed.st'
   local peak_model_filename            = basename .. '_peak.mod'
   local original_filename              = basename .. '_orig.st'
   local pre_xf_filename                = basename .. '.prexf'
   local pre_xg_filename                = basename .. '.prexg'
   local pre_aligned_filename           = basename .. '.preali'
   local fiducial_model_fixed_filename  = basename .. '_beadtrack.fid'
   local three_d_model_filename         = basename .. '.3dmod'
   local fiducial_xyz_filename          = basename .. 'fid.xyz'
   local residual_model_filename        = basename .. '.resid'
   local fiducial_text_model_filename   = basename .. '.fid.txt'
   local fiducial_xf_filename           = basename .. '_fid.xf'
   local xf_filename                    = basename .. '.xf'
   local fiducial_tilt_filename         = basename .. '_fid.tlt'
   local defocus_filename               = basename .. '.defocus'
   local first_aligned_filename         = basename .. '_first.ali'
   local ctf_corrected_aligned_filename = basename .. '_ctfcorr.ali'
   local xtilt_filename                 = basename .. '.xtilt'
   local gold_erase_model_filename      = basename .. '_erase.fid'
   local gold_erase_filename            = basename .. '_erase.ali'
   local second_aligned_filename        = basename .. '_second.ali'
   local RAPTOR_directory_name          = basename .. '_RAPTOR'
   local com_filenames                  = basename .. '_*.com'
   local log_filenames                  = basename .. '_*.log'
   local temporary_filenames            = basename .. '*~'

   pcall(os.execute, string.format(
         'rm -rf' .. string.rep(' %s ', 24),
         raw_tilt_filename,
         ccd_erased_filename,
         peak_model_filename,
         pre_xf_filename,
         pre_xg_filename,
         pre_aligned_filename,
         fiducial_model_fixed_filename,
         three_d_model_filename,
         fiducial_xyz_filename,
         residual_model_filename,
         fiducial_text_model_filename,
         fiducial_xf_filename,
         xf_filename,
         fiducial_tilt_filename,
{'fraction_non_alphanumeric': 0.17892824704813806, 'fraction_numerical': 0.0009082652134423251, 'mean_word_length': 1.8503019844693702, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2600968', 'n_tokens_mistral': 1860, 'n_tokens_neox': 1787, 'n_words': 367}