<commit_before>#!/bin/env/python3

import csv, os
from tkinter.filedialog import askopenfilename 

file = askopenfilename()
filename, file_extension = os.path.splitext(file)
file = open(file)

headEnd = 46
dataRows = []

def removeSet(data, Y1):
	result = []
	for row_num, row in enumerate(data):
		if row_num > 0 and row[1] == Y1:
			break
		result.append(row)
	if row_num == len(data)-1:
		row_num += 1
	data = data[row_num:]
	return result, data

with file as csvFileObj:
	readerObj = csv.reader(csvFileObj)

	# Take out all header rows
	for row in readerObj:
		if readerObj.line_num < headEnd:
			continue
		row = row[0].strip().split(" ") # Split into list
		# If the V value is positive it is separated by 2 spaces, leaving the data in list[2], this corrects that
		if row[1] == '':
			del row[1]
		dataRows.append(row[:2])

# Write a cleaned file with all runs in one set
big_out = filename + "_cleanBig" + file_extension
with open(big_out, "w") as f:
    f.write("A,V\n")
    for row in dataRows:
        f.write(row[0] + "," + row[1] + "\n")


Y1 = dataRows[0][1]
largest_data_set = 0
data_sets = []

# Create data sets by searching for Y1 in the data
while(len(dataRows) > 0):
    data_set, dataRows = removeSet(dataRows, Y1)
    data_sets.append(data_set)
    if len(data_set) > largest_data_set:
        largest_data_set = len(data_set)

# Make all of the data sets the same size
for i in range(len(data_sets)):
    pad_rows = largest_data_set - len(data_sets[i])
    for j in range(pad_rows):
        data_sets[i].append(["",""])


# Write a cleaned file with all runs separated
split_out = filename + "_cleanSplit" + file_extension
with open(split_out, "w") as f:
    # Write header row
    f.write("A,V")
    for i in range(1, len(data_sets)):
        f.write(",,A,V")
    f.write("\n")

    for row_num in range(largest_data_set):
        f.write(",".join(data_sets[0][row_num]))
        for i in range(1, len(data_sets)):
            f.write(",," + ",".join(data_sets[i][row_num]))
        f.write("\n")<commit_msg>Wrapped in main function to prep for multifile<commit_after>#!/bin/env/python3

import csv, os
from tkinter.filedialog import askopenfilename

def get_file():
	file = askopenfilename()
	filename, file_extension = os.path.splitext(file)
	file = open(file)
	file_info = [file, filename, file_extension]
	return file_info

headEnd = 46

# file_info = get_file()

def fill_array(file_info):
	with file_info[0] as file:
		data_rows = []
		readerObj = csv.reader(file)
		# Take out all header rows
		for row in readerObj:
			if readerObj.line_num < headEnd:
				continue
			row = row[0].strip().split(" ") # Split into list
			# If the V value is positive it is separated by 2 spaces, leaving the data in list[2], this corrects that
			if row[1] == '':
				del row[1]
			data_rows.append(row[:2])

	return data_rows

# data_rows = fill_array(file_info)

# Write a cleaned file with all runs in one set, i.e. as a 2*m matrix
def big_writer(file_info, data_rows):
	big_out = file_info[1] + "_cleanBig" + file_info[2]
	with open(big_out, "w") as f:
		f.write("A,V\n")
		for row in data_rows:
			f.write(row[0] + "," + row[1] + "\n")

# big_writer(file_info, data_rows)

def removeSet(data, initial_V):
	result = []
	for row_num, row in enumerate(data):
		if row_num > 0 and row[1] == initial_V:
			break
		result.append(row)
	if row_num == len(data)-1:
		row_num += 1
	data = data[row_num:]
	return result, data

# Write a cleaned file separated by runs
def run_parser(file_info, data_rows):
	initial_V = data_rows[0][1]
	largest_data_set = 0
	data_sets = []

	# Create data sets by searching for initial_V in the data
	while(len(data_rows) > 0):
		data_set, data_rows = removeSet(data_rows, initial_V)
		data_sets.append(data_set)
		if len(data_set) > largest_data_set:
			largest_data_set = len(data_set)

	# Make all of the data sets the same size
	for i in range(len(data_sets)):
		pad_rows = largest_data_set - len(data_sets[i])
		for j in range(pad_rows):
			data_sets[i].append(["",""])


	# Write a cleaned file with all runs separated
	split_out = file_info[1] + "_cleanSplit" + file_info[2]
	with open(split_out, "w") as f:
		# Write header row
		f.write("A,V")
		for i in range(1, len(data_sets)):
			f.write(",,A,V")
		f.write("\n")

		for row_num in range(largest_data_set):
			f.write(",".join(data_sets[0][row_num]))
			for i in range(1, len(data_sets)):
				f.write(",," + ",".join(data_sets[i][row_num]))
			f.write("\n")

# run_parser(data_rows)


def main():
	file_info = get_file()
	data_rows = fill_array(file_info)
	big_writer(file_info, data_rows)
	run_parser(file_info, data_rows)

if __name__ == '__main__':
	main()
{'fraction_non_alphanumeric': 0.13532060904996784, 'fraction_numerical': 0.011794981771391808, 'mean_word_length': 3.919831223628692, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1945956', 'n_tokens_mistral': 1841, 'n_tokens_neox': 1730, 'n_words': 477}