<commit_before># Import some goodies
import sys, json,os

#Import Authors, Quotes, and File Extensions
with open('db.json') as json_data:
    db = json.load(json_data)
    json_data.close()
smajson = db['fileext']

# Write to file
def write_file(stringy_string,namy_name):
    os.remove(namy_name)
    text_file = open(namy_name, "w")
    text_file.write(stringy_string)
    text_file.close()

# MaGiC
def boringify(filename):
	full_file = ""
	for line in open(filename):		
		if (line[0] != smajson[filename[filename.index('.')+1:]]):
			full_file = full_file + line
	write_file(full_file,filename)

	#Run the script on the output file
if (len(sys.argv) > 1):
    boringify(sys.argv[1])<commit_msg>indentation fix<commit_after># Import some goodies
import sys, json,os

#Import Authors, Quotes, and File Extensions
with open('db.json') as json_data:
    db = json.load(json_data)
    json_data.close()
smajson = db['fileext']

# Write to file
def write_file(stringy_string,namy_name):
    os.remove(namy_name)
    text_file = open(namy_name, "w")
    text_file.write(stringy_string)
    text_file.close()

# MaGiC
def boringify(filename):
	full_file = ""
	for line in open(filename):		
		if (line[0] != smajson[filename[filename.index('.')+1:]]):
			full_file = full_file + line
	write_file(full_file,filename)

#Run the script on the output file
if (len(sys.argv) > 1):
    boringify(sys.argv[1])
{'fraction_non_alphanumeric': 0.13457408732999285, 'fraction_numerical': 0.00572655690765927, 'mean_word_length': 4.706122448979592, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7503593', 'n_tokens_mistral': 530, 'n_tokens_neox': 518, 'n_words': 118}