<commit_before>#(C) 2017 <NAME>

import zlib
import sys
import os

class color:
	HEADER = '\033[95m'
	OKBLUE = '\033[94m'
	OKGREEN = '\033[92m'
	WARNING = '\033[93m'
	FAIL = '\033[91m'
	ENDC = '\033[0m'
	BOLD = '\033[1m'
	UNDERLINE = '\033[4m'


def split(string, splitters):
	final = [string]
	for x in splitters:
		for i,s in enumerate(final):
			if x in s and x != s:
				left, right = s.split(x, 1)
				final[i] = left
				final.insert(i + 1, x)
				final.insert(i + 2, right)
	return final


def crc(filepath):
	previous = 0
	try:
		for line in open(filepath,"rb"):
			previous = zlib.crc32(line, previous)
	except OSError:
		print("CRC ERROR: OSERROR")
	return "%X"%(previous & 0xFFFFFFFF)


def scandirectory(walk_dir, scanfile, verbose = False):
	try:
		current_scan = []
		for root, subdirs, files in os.walk(walk_dir):
			#current_scan.extend([f"{root}\n"])
			for filename in files:
				if filename == "media-validation-tool" or filename == "media.csv" or filename == "media-validation-tool.py" or filename == "." or filename == "..":
					break
				else:
					if os.name == "posix":
						file_path = root + "/" + filename
					elif os.name == "nt":
						file_path = root + "\\" + filename
					else:
						pass
					checksum = crc(os.path.join(root, filename))
					current_scan.extend([file_path + "," + checksum + ",\n"])
		with open(scanfile, "w") as current_scan_file:
			current_scan_file.writelines(current_scan)			
	except FileNotFoundError:
		if verbose == True:
			print("SCAN ERROR: FILE NOT FOUND")

def validate(path = ".", verifile = "./media.csv"):
	print("Validating " + path)
	f = open(verifile, "r")
	verifileslist = f.readlines()
	scandirectory(".", "media.csv.1")
	fold = open("media.csv", "r")
	fnew = open("media.csv.1", "r")
	foldarray = fold.readlines()
	fnewarray = fnew.readlines()
	errorcount = 0
	successcount = 0
	for line in foldarray:
		splitline = split(line, ",")
		if not line in fnewarray:
			print(color.FAIL + "ERROR   " + color.ENDC + splitline[0])
			errorcount = errorcount + 1
		else:
			print(color.OKGREEN + "SUCCESS " + color.ENDC + splitline[0])
			successcount = successcount + 1
	print(color.BOLD, errorcount, "errors in", successcount + errorcount, "files", color.ENDC)
	fold.close()
	fnew.close()
	os.remove("media.csv.1")

def create(path = ".", verifile = "./media.csv"):
	print("Creating media file...")
	scandirectory(".", "media.csv")
	
def main():
	print("MEDIA VALIDATION TOOL")
	print("(C) 2017 <NAME>")
	if len(sys.argv) < 2:
		validate()
		input("Press ENTER to continue...")
	else:
		create()
	
	

if __name__ == "__main__":
	main()

<commit_msg>Remove color features because of windows...<commit_after>#(C) 2017 <NAME>

show_success = True

import zlib
import sys
import os

class color:
	HEADER = '\033[95m'
	OKBLUE = '\033[94m'
	OKGREEN = '\033[92m'
	WARNING = '\033[93m'
	FAIL = '\033[91m'
	ENDC = '\033[0m'
	BOLD = '\033[1m'
	UNDERLINE = '\033[4m'


def split(string, splitters):
	final = [string]
	for x in splitters:
		for i,s in enumerate(final):
			if x in s and x != s:
				left, right = s.split(x, 1)
				final[i] = left
				final.insert(i + 1, x)
				final.insert(i + 2, right)
	return final


def crc(filepath):
	previous = 0
	try:
		for line in open(filepath,"rb"):
			previous = zlib.crc32(line, previous)
	except OSError:
		print("CRC ERROR: OSERROR")
	return "%X"%(previous & 0xFFFFFFFF)


def scandirectory(walk_dir, scanfile, verbose = False):
	try:
		current_scan = []
		for root, subdirs, files in os.walk(walk_dir):
			#current_scan.extend([f"{root}\n"])
			for filename in files:
				if filename == "media-validation-tool" or filename == "media.csv" or filename == "media-validation-tool.py" or filename == "." or filename == "..":
					break
				else:
					if os.name == "posix":
						file_path = root + "/" + filename
					elif os.name == "nt":
						file_path = root + "\\" + filename
					else:
						pass
					checksum = crc(os.path.join(root, filename))
					current_scan.extend([file_path + "," + checksum + ",\n"])
		with open(scanfile, "w") as current_scan_file:
			current_scan_file.writelines(current_scan)			
	except FileNotFoundError:
		if verbose == True:
			print("SCAN ERROR: FILE NOT FOUND")

def validate(path = ".", verifile = "./media.csv"):
	print("Validating " + path)
	f = open(verifile, "r")
	verifileslist = f.readlines()
	scandirectory(".", "media.csv.1")
	fold = open("media.csv", "r")
	fnew = open("media.csv.1", "r")
	foldarray = fold.readlines()
	fnewarray = fnew.readlines()
	errorcount = 0
	successcount = 0
	for line in foldarray:
		splitline = split(line, ",")
		if not line in fnewarray:
			print("* ERROR " + splitline[0])
			errorcount = errorcount + 1
		else:
			if show_success == True:
				print("SUCCESS " + splitline[0])
			successcount = successcount + 1
	print(errorcount, "errors in", successcount + errorcount, "files")
	fold.close()
	fnew.close()
	os.remove("media.csv.1")

def create(path = ".", verifile = "./media.csv"):
	print("Creating media file...")
	scandirectory(".", "media.csv")
	
def main():
	print("MEDIA VALIDATION TOOL")
	print("(C) 2017 <NAME>")
	if len(sys.argv) < 2:
		validate()
		input("Press ENTER to continue...")
	else:
		create()
	
	

if __name__ == "__main__":
	main()


{'fraction_non_alphanumeric': 0.16644397785836992, 'fraction_numerical': 0.023668639053254437, 'mean_word_length': 3.4863013698630136, 'pattern_counts': {'":': 8, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5290990', 'n_tokens_mistral': 2228, 'n_tokens_neox': 1933, 'n_words': 495}