<commit_before>module ProcessingFramework
	require "clamp"
	require "fileutils"
	class CommandLineHelper  < Clamp::Command
		# shared options
		#@config = nil
  		option ["-b", "--basename"], "basename", "The basename of the data to be processed. For example npp.14300.1843 .  Appended to --outputdir if included."
#		option ["-i", "--input"], "input", "The source directory. If --basename is included, it is appended to this.", :required => true
		option ["-o", "--outdir"], "outdir", "The output directory. If --basename is included, it is appended to this.", :required => true
		option ["-t", "--tempdir"], "tempdir", "The temp directory, used for working space. A sub directory will made inside this named after the basename.",  :environment_variable => "PROCESSING_TEMPDIR", :required => true

  		def execute
			raise "CommandLineHelper should not be instatiated directly."
    		end


		#Perhaps not the best place for this - copies everything in the current directory to the directory at output
		def copy_output( output, save_glob="*")
			#add trailing slash, if needed
			output += "/" if output[-1] != "/"
		
			FileUtils.mkdir_p(output)  if (!File.exists?(output))
                        Dir.glob(save_glob).each do |x|
				if ( File?(x) )
					puts("INFO: Copying #{x} to #{output}")
                                	FileUtils.cp(x, output)
				else
					puts("INFO: Not a file, skipping #{x} to #{output}")
				end
                        end
		end

  	end
end
<commit_msg>Silly typo fix.  Now works.<commit_after>module ProcessingFramework
	require "clamp"
	require "fileutils"
	class CommandLineHelper  < Clamp::Command
		# shared options
		#@config = nil
  		option ["-b", "--basename"], "basename", "The basename of the data to be processed. For example npp.14300.1843 .  Appended to --outputdir if included."
#		option ["-i", "--input"], "input", "The source directory. If --basename is included, it is appended to this.", :required => true
		option ["-o", "--outdir"], "outdir", "The output directory. If --basename is included, it is appended to this.", :required => true
		option ["-t", "--tempdir"], "tempdir", "The temp directory, used for working space. A sub directory will made inside this named after the basename.",  :environment_variable => "PROCESSING_TEMPDIR", :required => true

  		def execute
			raise "CommandLineHelper should not be instatiated directly."
    		end


		#Perhaps not the best place for this - copies everything in the current directory to the directory at output
		def copy_output( output, save_glob="*", copy_dirs=false)
			#add trailing slash, if needed
			output += "/" if output[-1] != "/"
		
			FileUtils.mkdir_p(output)  if (!File.exists?(output))
                        Dir.glob(save_glob).each do |x|
				if ( File.file?(x) || copy_dirs )
					puts("INFO: Copying #{x} to #{output}")
                                	FileUtils.cp_r(x, output)
				else
					puts("INFO: Not a file, skipping #{x} to #{output}")
				end
                        end
		end

  	end
end

{'fraction_non_alphanumeric': 0.12917079616782293, 'fraction_numerical': 0.006607201850016518, 'mean_word_length': 2.922279792746114, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4732112', 'n_tokens_mistral': 981, 'n_tokens_neox': 888, 'n_words': 334}