<commit_before>import glob
import os
import subprocess

"""This is used to decompress the news.bin files."""

def decompress(file):
	with open(file, "rb") as source_file:
		read = source_file.read()
		
		tail = read[320:]
		
	with open(file + ".2", "w+") as dest_file:	
		dest_file.write(tail)
		
	FNULL = open(os.devnull, "w+")
		
	decompress = subprocess.call(["mono", "--runtime=v4.0.30319", "DSDecmp.exe", "-d", file + ".2", file + ".3"], stdout=FNULL, stderr=subprocess.STDOUT)
	remove = os.remove(file + ".2")
	move = subprocess.call(["mv", file + ".3", file + ".2"], stdout=FNULL, stderr=subprocess.STDOUT)
	open_hex = subprocess.call(["open", "-a", "Hex Fiend", file + ".2"], stdout=FNULL, stderr=subprocess.STDOUT)
	
for file in glob.glob("news.bin.*"):
	if os.path.exists(file):
		decompress(file)
	
for file in glob.glob("*.bin"):
	if os.path.exists(file):
		decompress(file)<commit_msg>Comment about the hex part<commit_after>import glob
import os
import subprocess

"""This is used to decompress the news.bin files."""

def decompress(file):
	with open(file, "rb") as source_file:
		read = source_file.read()
		
		tail = read[320:]
		
	with open(file + ".2", "w+") as dest_file:	
		dest_file.write(tail)
		
	FNULL = open(os.devnull, "w+")
		
	decompress = subprocess.call(["mono", "--runtime=v4.0.30319", "DSDecmp.exe", "-d", file + ".2", file + ".3"], stdout=FNULL, stderr=subprocess.STDOUT)
	remove = os.remove(file + ".2")
	move = subprocess.call(["mv", file + ".3", file + ".2"], stdout=FNULL, stderr=subprocess.STDOUT)
	open_hex = subprocess.call(["open", "-a", "Hex Fiend", file + ".2"], stdout=FNULL, stderr=subprocess.STDOUT) // This is to open the news files in the Mac hex editor I use called Hex Fiend.
	
for file in glob.glob("news.bin.*"):
	if os.path.exists(file):
		decompress(file)
	
for file in glob.glob("*.bin"):
	if os.path.exists(file):
		decompress(file)
{'fraction_non_alphanumeric': 0.18793012175754367, 'fraction_numerical': 0.017998941238750663, 'mean_word_length': 4.279329608938547, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7562052', 'n_tokens_mistral': 752, 'n_tokens_neox': 701, 'n_words': 180}