<commit_before>import csv
import os

#return: dict, key = uniq id
#val: dict, key = column name, val = val
#example: dict: {2066053: {'affiliation': 'KAIST', 'name': 'myname'}}
def load_single_file(input_file):
	with open(input_file, 'r', encoding='utf-8') as read_file:
		reader = csv.reader(read_file)
		column = reader.__next__()
		column = [x.strip().lower() for x in column]

		id_table = dict()

		for line in reader:
			if(len(line) == 0):
				continue
			val = dict()
			key = None
			prep_key = []
			for (col, item) in zip(column, line):
				if('id' in col):
					prep_key.append(int(item))
					continue
				val[col] = item
			if(len(prep_key) == 1):
				key = prep_key[0]
			else:
				key = tuple(prep_key)
			id_table[key] = val
		return id_table

def load_all(directory = '.', file_list = None):
	return_data = dict()
	if file_list == None:
		file_list = ['Author', 'Conference', 'Journal', 'PaperAuthor', 'Test', 'Train', 'Valid', 'ValidSolution']
	for name in file_list:
		file_name = os.path.join(directory, '{}.csv'.format(name))
		return_data[name] = load_single_file(file_name)
	return return_data<commit_msg>fix loader to accept non intergers<commit_after>import csv
import os

#return: dict, key = uniq id
#val: dict, key = column name, val = val
#example: dict: {2066053: {'affiliation': 'KAIST', 'name': 'myname'}}
def load_single_file(input_file):
	with open(input_file, 'r', encoding='utf-8') as read_file:
		reader = csv.reader(read_file)
		column = reader.__next__()
		column = [x.strip().lower() for x in column]

		id_table = dict()

		for line in reader:
			if(len(line) == 0):
				continue
			val = dict()
			key = None
			prep_key = []
			for (col, item) in zip(column, line):
				if('id' in col):
					if item.isdigit():
						prep_key.append(int(item))
					else:
						prep_key.append(str(item))
					continue
				val[col] = item
			if(len(prep_key) == 1):
				key = prep_key[0]
			else:
				key = tuple(prep_key)
			id_table[key] = val
		return id_table

def load_all(directory = '.', file_list = None):
	return_data = dict()
	if file_list == None:
		file_list = ['Author', 'Conference', 'Journal', 'PaperAuthor', 'Test', 'Train', 'Valid', 'ValidSolution']
	for name in file_list:
		file_name = os.path.join(directory, '{}.csv'.format(name))
		return_data[name] = load_single_file(file_name)
	return return_data
{'fraction_non_alphanumeric': 0.1567291311754685, 'fraction_numerical': 0.009369676320272573, 'mean_word_length': 3.9245283018867925, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6284900', 'n_tokens_mistral': 995, 'n_tokens_neox': 849, 'n_words': 224}