<commit_before>use ooc-base
use ooc-collections
import io/[File, FileReader]

// TODO: Handle multi-line string literals

CsvReader: class extends Iterator<VectorList<Text>> {
	_fileReader: FileReader
	init: func (=_fileReader)
	free: func {
		if (this _fileReader != null) {
			this _fileReader close()
			this _fileReader free()
		}
		super()
	}
	remove: func -> Bool { false }
	iterator: func -> This { this }
	hasNext?: func -> Bool { this _fileReader hasNext?() }
	next: func -> VectorList<Text> {
		result: VectorList<Text>
		if (this hasNext?()) {
			readCharacter: Char
			textBuilder := TextBuilder new()
			while (this _fileReader hasNext?() && ((readCharacter = this _fileReader read()) != '\n' && readCharacter != '\0'))
				textBuilder append(readCharacter)
			result = this _parseRow(textBuilder toText())
			textBuilder free()
		}
		result
	}
	_parseRow: func (rowData: Text) -> VectorList<Text> {
		result := VectorList<Text> new()
		rowDataLength := rowData count
		readCharacter: Char
		for (i in 0 .. rowDataLength) {
			textBuilder := TextBuilder new()
			while (i < rowDataLength && ((readCharacter = rowData take()[i]) != This delimiter)) {
				++i
				match (readCharacter) {
					case ' ' =>
						continue
					case '\t' =>
						continue
					case '\r' =>
						continue
					case '"' =>
						textBuilder append(this _extractStringLiteral(rowData take(), i&))<commit_msg>Changed to use take in one place.<commit_after>use ooc-base
use ooc-collections
import io/[File, FileReader]

// TODO: Handle multi-line string literals

CsvReader: class extends Iterator<VectorList<Text>> {
	_fileReader: FileReader
	init: func (=_fileReader)
	free: func {
		if (this _fileReader != null) {
			this _fileReader close()
			this _fileReader free()
		}
		super()
	}
	remove: func -> Bool { false }
	iterator: func -> This { this }
	hasNext?: func -> Bool { this _fileReader hasNext?() }
	next: func -> VectorList<Text> {
		result: VectorList<Text>
		if (this hasNext?()) {
			readCharacter: Char
			textBuilder := TextBuilder new()
			while (this _fileReader hasNext?() && ((readCharacter = this _fileReader read()) != '\n' && readCharacter != '\0'))
				textBuilder append(readCharacter)
			result = this _parseRow(textBuilder toText())
			textBuilder free()
		}
		result
	}
	_parseRow: func (rowData: Text) -> VectorList<Text> {
		row := rowData take()
		result := VectorList<Text> new()
		rowLength := row count
		readCharacter: Char
		for (i in 0 .. rowLength) {
			textBuilder := TextBuilder new()
			while (i < rowLength && ((readCharacter = row[i]) != This delimiter)) {
				++i
				match (readCharacter) {
					case ' ' =>
						continue
					case '\t' =>
						continue
					case '\r' =>
						continue
					case '"' =>
						textBuilder append(this _extractStringLiteral(row, i&))
{'fraction_non_alphanumeric': 0.13668807994289792, 'fraction_numerical': 0.0014275517487508922, 'mean_word_length': 3.463375796178344, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '320950', 'n_tokens_mistral': 1036, 'n_tokens_neox': 878, 'n_words': 260}