<commit_before>require './QCscript'

input = "/Users/mbell/Science/current/eldora/autoqc/test/ori"
output = "/Users/mbell/Science/current/eldora/autoqc/test/qced"
ta_cfac = "/Users/mbell/Science/current/eldora/autoqc/test/rf12.cfac.aft"
tf_cfac = "/Users/mbell/Science/current/eldora/autoqc/test/rf12.cfac.fore"

qc = QCscript.new
qc.setInputPath(input)
qc.setOutputPath(output)
puts 'Processing data from ' + qc.getInputPath + ' to ' + qc.getOutputPath
0.upto(qc.getFileListSize - 1) { |file|
	puts "Processing #{file}"
	qc.load(file)

	qc.setNavigationCorrections(ta_cfac, "TA-ELDR")
        qc.setNavigationCorrections(tf_cfac, "TF-ELDR")
	qc.removeAircraftMotion("VR", "VQC");

	qc.thresholdData("NCP", "ZZ", 0.2, "below")
	qc.thresholdData("NCP","VQC", 0.2, "below")

	qc.probGroundGates("ZZ", "GG", 2.0);
	qc.thresholdData("GG","ZZ", 0.7, "above");
	qc.thresholdData("GG","VQC", 0.7, "above");
		
	qc.calcRatio("SW", "ZZ", "SWZ", true);
	qc.thresholdData("SWZ","ZZ", 0.6, "above");
	qc.thresholdData("SWZ","VQC", 0.6, "above");

	qc.despeckleRadial("ZZ", 2);
	qc.despeckleRadial("VQC", 2);
	qc.despeckleAzimuthal("ZZ", 2);
	qc.despeckleAzimuthal("VQC", 2);

	qc.save(file)
}
puts 'QC complete'
<commit_msg>Minor clean up of defaultQC.rb<commit_after>require './QCscript'

input = "/Users/mbell/Science/current/eldora/autoqc/test/ori"
output = "/Users/mbell/Science/current/eldora/autoqc/test/qced"
ta_cfac = "/Users/mbell/Science/current/eldora/autoqc/test/rf12.cfac.aft"
tf_cfac = "/Users/mbell/Science/current/eldora/autoqc/test/rf12.cfac.fore"

qc = QCscript.new
qc.setInputPath(input)
qc.setOutputPath(output)
puts "Processing data from #{qc.getInputPath} to #{qc.getOutputPath}"
qc.getFileListSize.times { |file|
	puts "Processing file #{file}"
	qc.instance_eval do
	   load file

	   setNavigationCorrections(ta_cfac, "TA-ELDR")
           setNavigationCorrections(tf_cfac, "TF-ELDR")
	   removeAircraftMotion("VR", "VQC");

	   thresholdData("NCP", "ZZ", 0.2, "below")
	   thresholdData("NCP","VQC", 0.2, "below")

	   probGroundGates("ZZ", "GG", 2.0);
	   thresholdData("GG","ZZ", 0.7, "above");
	   thresholdData("GG","VQC", 0.7, "above");
		
	   calcRatio("SW", "ZZ", "SWZ", true);
	   thresholdData("SWZ","ZZ", 0.6, "above");
	   thresholdData("SWZ","VQC", 0.6, "above");

	   despeckleRadial("ZZ", 2);
	   despeckleRadial("VQC", 2);
	   despeckleAzimuthal("ZZ", 2);
	   despeckleAzimuthal("VQC", 2);

	   save file
	end
}
puts 'QC complete'

{'fraction_non_alphanumeric': 0.1956079707198048, 'fraction_numerical': 0.01870679137860919, 'mean_word_length': 4.774647887323944, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '856670', 'n_tokens_mistral': 1131, 'n_tokens_neox': 979, 'n_words': 153}