<commit_before>import os
import glob
path = "./*.csv"

initialChar = ","
replaceChar = "|"
removeQuotes = True
savePath = "./fixed/" #Directory currently needs to exist first

for fname in glob.glob(path):
	print(os.path.basename(fname))
	with open(fname) as csvFile:
		with open(savePath+fname, 'w') as fixedcsv:
			for line in csvFile:
				#Split line in to a set of characters
				lineList = list(line)

				notInQuotes = True #Because we will always start in an unquoted context.

				charNum = len(lineList)

				i = 0

				while (i < charNum):
					if lineList[i] == initialChar and notInQuotes:
						lineList[i] = replaceChar

					if lineList[i] == '"':
						if notInQuotes:
							if removeQuotes:
								lineList[i] = ''
							notInQuotes = False
						elif not notInQuotes:
							if removeQuotes:
								lineList[i] = ''
							notInQuotes = True

					i += 1

				print("".join(lineList))
				fixedcsv.write("".join(lineList))

	
<commit_msg>Fixed multi-line quoted text issue<commit_after>import os
import csv
import glob
path = "./*.csv"

initialChar = ","
replaceChar = "|"
removeQuotes = True
savePath = "./fixed/" #Directory currently needs to exist first
tempString = "temp_"

for fname in glob.glob(path):
	print("Processing "+os.path.basename(fname))
	#Remove any additional line breaks that appear in quoted text
	tempFile = savePath+tempString+os.path.basename(fname)
	with open(fname, "rb") as input, open(savePath+tempString+os.path.basename(fname), "wb") as output:
		w = csv.writer(output)
		for record in csv.reader(input):
			w.writerow(tuple(s.replace("\r\n","") for s in record))
		
	with open(tempFile) as csvFile:
		with open(savePath+fname, 'w') as fixedcsv:
			for line in csvFile:
				#Split line in to a set of characters
				lineList = list(line)
				notInQuotes = True #Because we will always start in an unquoted context.
				charNum = len(lineList)
				i = 0
				while (i < charNum):
					if lineList[i] == initialChar and notInQuotes:
						lineList[i] = replaceChar
					if lineList[i] == '"':
						if notInQuotes:
							if removeQuotes:
								lineList[i] = ''
							notInQuotes = False
						elif not notInQuotes:
							if removeQuotes:
								lineList[i] = ''
							notInQuotes = True

					i += 1

				fixedcsv.write("".join(lineList))
	print(os.path.basename(fname)+" Done")
	os.remove(tempFile)

{'fraction_non_alphanumeric': 0.12011884550084889, 'fraction_numerical': 0.001697792869269949, 'mean_word_length': 3.0991304347826087, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3050497', 'n_tokens_mistral': 994, 'n_tokens_neox': 782, 'n_words': 227}