<commit_before><commit_msg>first commit of pack.py (not working yet by any means)<commit_after>import os
import argparse
import zlib
import sys

def main():
	parser = argparse.ArgumentParser()
	parser.add_argument('--input', dest='input', required=False, default=os.getcwd())
	parser.add_argument('--output', dest='output', required=False, default=os.getcwd())
	args = parser.parse_args()

	if not os.path.isdir(args.input):
		parser.print_help()

	if not os.path.isdir(args.output):
		parser.print_help()

	resources = set()

	for root, dirs, files in os.walk(args.input, topdown=False):
	    for file in files:
	    	if file in resources:
	    		print 'duplicate file name (' + file + '), terminating'
	    		sys.exit(1)

	    	print file

	    	resources.add(file)

	    	with open(os.path.join(root, file), 'rb') as f:
	    		crc32 = zlib.crc32(f.read())

if __name__ == '__main__':
	main()
{'fraction_non_alphanumeric': 0.14317673378076062, 'fraction_numerical': 0.005592841163310962, 'mean_word_length': 3.8118279569892475, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7441022', 'n_tokens_mistral': 330, 'n_tokens_neox': 308, 'n_words': 78}