<commit_before>package common

func parseLine(line string) []interface{} {
	// TODO
}
<commit_msg>Adds more parse functions<commit_after>package common

import (
	"bufio"
	"io"
	"strconv"
)

const (
	delim = ' '
	quote = '"'
	literalStart = '{'
	literalEnd = '}'
	listStart = '('
	listEnd = ')'
)

type Literal struct {
	Len int
	Str string
}

func (l *Literal) String() string {
	return l.Str
}

func (l *Literal) WriteTo(w io.Writer) (N int64, err error) {
	n, err := io.WriteString(w, string(literalStart) + strconv.Itoa(l.Len) + string(literalEnd))
	return int64(n), err
}

func trimSuffix(str string, suffix rune) string {
	return str[:len(str)-1]
}

func parseAtom(r bufio.Reader) (interface{}, error) {
	atom, err := r.ReadString(byte(delim))
	if err != nil && err != io.EOF {
		return nil, err
	}
	atom = trimSuffix(atom, delim)

	if atom == "NIL" {
		return nil, nil
	}
	return atom, nil
}

func parseLiteral(r bufio.Reader) (literal *imap.Literal, err error) {
	char, _, err := r.ReadRune()
	if err != nil {
		return
	}
	if char != literalStart {
		err = errors.New("Literal string doesn't start with an open brace")
		return
	}

	lstr, err := r.ReadString(byte(literalEnd))
	if err != nil {
		return
	}
	lstr = trimSuffix(lstr, literalEnd)

	l, err := strconv.Atoi(lstr)
	if err != nil {
		return
	}

	literal = &Literal{Len: l}
	return
}

func parseQuotedString(r bufio.Reader) (str string, err error) {
	char, _, err := r.ReadRune()
	if err != nil {
		return
	}
	if char != quote {
		err = errors.New("Quoted string doesn't start with a quote")
		return
	}

	str, err := r.ReadString(byte(quote))
	if err != nil {
		return
	}
	str = trimSuffix(str, quote)
	return
}

func parseList(r bufio.Reader) (fields []interface{}, err error) {
	return // TODO
}

func parseLine(r bufio.Reader) (fields []interface{}, err error) {
	var char rune
	for {
		chars, err = r.Peek(1)
		if err != nil {
			return
		}
		char = rune(chars[0])

		var field interface{}
		switch char {
		case '\n':
			return
		case literalStart:
			field, err = parseLiteral(r)
		case quote:
			field, err = parseQuotedString(r)
		case listStart:
			field, err = parseList(r)
		default:
			field, err = parseAtom(r)
		}

		if err != nil {
			return
		}

		fields = append(fields, field)
	}
}

{'fraction_non_alphanumeric': 0.1446846052051169, 'fraction_numerical': 0.0030877812086457872, 'mean_word_length': 3.4210526315789473, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4176797', 'n_tokens_mistral': 914, 'n_tokens_neox': 858, 'n_words': 239}