<commit_before>import os
import hashlib

def hashy(file):
	hasher = hashlib.md5()
	with open(file, 'rb') as afile:
		buf = afile.read()
		hasher.update(buf)
		return hasher.hexdigest()

fileHashes = []
def checkIfAlreadyExists(hash):
	if hash in fileHashes:
		return True
	else:
		return False

deleted = 0
def shouldIDeleteThisFile(fileName):
	global deleted
	hash = hashy(fileName)
	if checkIfAlreadyExists(hash):
		os.remove(fileName)
		print(hash + ' : ' + fileName + ' - Deleted')
		deleted += 1
	else:
		fileHashes.append(hash)
		print(hash + ' : ' + fileName + ' - Added to list')

totalNumberOfFiles = 0
for (dir, _, files) in os.walk(os.curdir):
	for f in files:
		path = os.path.join(dir, f)
		totalNumberOfFiles += 1
		shouldIDeleteThisFile(path)

print('Finished. ' + str(deleted) + '/' + str(totalNumberOfFiles) + ' files were duplicates. They have been deleted.')
<commit_msg>Update dupey.py<commit_after>import os
import hashlib

def hashy(file):
	hasher = hashlib.md5()
	with open(file, 'rb') as afile:
		buf = afile.read()
		hasher.update(buf)
		return hasher.hexdigest()

fileHashes = []
def checkIfAlreadyExists(hash):
	if hash in fileHashes:
		return True
	else:
		return False

deleted = 0
def shouldIDeleteThisFile(fileName):
	global deleted
	hash = hashy(fileName)
	if checkIfAlreadyExists(hash):
		os.remove(fileName)
		print hash + ' : ' + fileName + ' - Deleted'
		deleted += 1
	else:
		fileHashes.append(hash)
		print hash + ' : ' + fileName + ' - Added to list'

totalNumberOfFiles = 0
for (dir, _, files) in os.walk(os.curdir):
	for f in files:
		path = os.path.join(dir, f)
		totalNumberOfFiles += 1
		shouldIDeleteThisFile(path)

print 'Finished. ' + str(deleted) + '/' + str(totalNumberOfFiles) + ' files were duplicates. They have been deleted.'

{'fraction_non_alphanumeric': 0.12759977515458124, 'fraction_numerical': 0.005621135469364812, 'mean_word_length': 4.144508670520231, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4052831', 'n_tokens_mistral': 679, 'n_tokens_neox': 648, 'n_words': 164}