<commit_before><commit_msg>Add the basic version of my file comparison script<commit_after>#! /usr/env/python
import os
import hashlib
import sys

bufsize = 65536
# Path1 = '/Users/kirkchambers/Desktop'
# Path2 = '/Users/kirkchambers/DataSets'

def generate_file_digests_for(path):
	path_set = set()
	for item in os.walk(path):
		(directory, _subdirectories, files) = item
		for file in files:
			if (file[0] == '.'):
				continue
			else:
				fqFilename = os.path.join(directory, file)
				path_set.add(generate_file_digest(fqFilename, file))
	return path_set


def generate_file_digest(fqFilename, shortFilename):
	hasher = hashlib.md5()
	with open(fqFilename, 'rb') as filestream:
		fileBuffer = filestream.read(bufsize)
		while len(fileBuffer) > 0:
			hasher.update(fileBuffer)
			fileBuffer = filestream.read(bufsize)
	# return "Filename:{file}\nHash:{hash}\nSize:{size}\n".format(file=fqFilename, hash=hasher.hexdigest(), size=os.path.getsize(fqFilename))
	return (hasher.hexdigest(), fqFilename, os.path.getsize(fqFilename))


def usage():
	print "file_list.py directory1 directory2"
	print "Prints out the files present in directory1 which are NOT present in directory2"


if __name__ == "__main__":
	try:
		(_command, Path1, Path2) = sys.argv
	except:
		usage()
		exit(1)

	path_set_1 = generate_file_digests_for(Path1)
	path_set_2 = generate_file_digests_for(Path2)

	# union = path_set_1 | path_set_2
	set_1_exclusives = path_set_1 - path_set_2
	# set_2_exclusives = path_set_2 - path_set_1
	# print "length of 1: {}".format(len(path_set_1))
	# print "length of 2: {}".format(len(path_set_2))
	# print "length of union: {}".format(len(union))
	# print "length of set1 uniqs: {}".format(len(set_1_exclusives))
	# print "length of set2 uniqs: {}".format(len(set_2_exclusives))

	print "Files present in {path1} and not in {path2}:".format(path1=Path1, path2=Path2)
	for item in set_1_exclusives:
		print item[1]
{'fraction_non_alphanumeric': 0.12760416666666666, 'fraction_numerical': 0.0234375, 'mean_word_length': 5.277777777777778, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '742803', 'n_tokens_mistral': 760, 'n_tokens_neox': 712, 'n_words': 162}