<commit_before>package edn

import (
	"bytes"
	"bufio"
	"io"
	"strings"
	"math/big"
	"unicode"
)

type readFn func(io.Reader, rune) interface{}

func Read(rdr bufio.Reader) (interface{}, error) {
	for {
		ch, _, err := rdr.ReadRune()
		if err != nil {
			return nil, err
		}
		if unicode.IsDigit(ch) {
			return readNum(r, ch)
		}
		
			<commit_msg>much more progress on the reader, now need to make a few decisions (lex ourselves, or use regexps; interface with strconv to help convert, etc.)<commit_after>package edn

import (
	"bytes"
	"bufio"
	"fmt"
	"io"
	"regexp"
	"strings"
	"math/big"
	"unicode"
)

type readFn func(io.Reader, rune) (interface{}, error)

var (
	macros = map[rune]readFn{
		'"' : stringRead,
		';' : commentRead,
		'(' : listRead,
		'[' : vectorRead,
		'{' : mapRead,
		'\\' : charRead,
		'#' : dispatchRead,
	}
	symPat = regexp.MustCompile(`:?([^/0-9].*/)?(/|[^/0-9][^/]*)`)
)

func ReadStr(string src) (interface{}, error) {
	return Read(strings.NewReader(src))
}

func Read(r io.Reader) (interface{}, error) {
	rdr := bufio.NewReader(r)
	for {
		ch, _, err := rdr.ReadRune()
		if err != nil { return nil, err }
		for unicode.IsSpace(ch) {
			ch, _, err := rdr.ReadRune()
			if err != nil { return nil, err }
		}
		if unicode.IsDigit(ch) {
			return readNum(r, ch)
		}
		if fn, ok := macros[ch]; ok {
			ret, err := fn(rdr, ch)
			if ret == rdr {
				continue
			}
			return ret, err
		}
		if ch == '+' || ch == '-' {
			ch2, _, err := rdr.ReadRune()
			if err != nil { return nil, err }
			// since we have the char, send it back in
			// before we continue
			err = rdr.UnreadRune() 
			if err != nil { return nil, err }
			if unicode.IsDIgit(ch2) { // number
				return readNum(r, ch)
			}
			
		}
		// true to check leading char
		return interpret(readToken(r, ch, true))
	}
}

func invalidRune(ch rune) bool { return ch == '`' || ch == '~' || ch == '@' }

func readToken(rdr bufio.Reader, ch rune, lead bool) (string, error) {
	if lead && invalidRune(ch) {
		return "", fmt.Errorf("Invalid leading character: %c", ch)
	}
	var buf bytes.Buffer
	buf.WriteRune(ch)
	for {
		ch, _, err := rdr.ReadRune()
		if err != nil || unicode.IsSpace(ch) || isTermMacro(ch) {
			rdr.UnreadRune()
			return buf.String()
		} else if invalidRune(ch) {
			return "", fmt.Errorf("Invalid constituent character: %c", ch)
		}
		buf.WriteRune(ch)
	}
}

func readNum(rdr bufio.Reader, ch rune) (interface{}, error) {
	var buf bytes.Buffer
	buf.WriteRune(ch)
	for {
		ch, _, err := rdr.ReadRune()
		if err != nil || unicode.IsSpace(ch) || isMacro(ch) {
			rdr.UnreadRune()
			break
		}
		buf.WriteRune(ch)
	}

	return matchNum(buf.String())
}

{'fraction_non_alphanumeric': 0.17409365558912387, 'fraction_numerical': 0.0022658610271903325, 'mean_word_length': 3.119751166407465, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4681944', 'n_tokens_mistral': 1157, 'n_tokens_neox': 1049, 'n_words': 267}