<commit_before>package bencode

import (
	"strings"
	"fmt"
	"strconv"
)

type token struct {
	typ tokenType
	val string
}

func (t token) String() string {
	return fmt.Sprintf("[%s]%q", t.typ, t.val)
}

type tokenType int

const (
	eofType = tokenType(iota)
	errorType
	stringType
	intType
	dictStartType
	dictEndType
	listStartType
	listEndType
)

var tokenNames = map[tokenType]string{
	eofType:       "EOF",
	errorType:     "error",
	stringType:    "string",
	intType:       "int",
	dictStartType: "dictStart",
	dictEndType:   "dictEnd",
	listStartType: "listStart",
	listEndType:   "listEnd",
}

func (t tokenType) String() string {
	return tokenNames[t]
}

type tokenStack []tokenType

func (ts *tokenStack) push(t tokenType) {
	*ts = append(tokenStack{t}, *ts...)
}
func (t *tokenStack) pop() tokenType {
	val := (*t)[0]
	*t = (*t)[1:]
	return val
}

const eof = -1

type stateFn func(*lexer) stateFn

type lexer struct {
	input      string
	state      stateFn
	pos        int
	start      int
	width      int
	items      chan token
	peekBuffer *token
	eofd       bool
	stack      tokenStack
}

func (l *lexer) next() (rune int) {
	if l.pos >= len(l.input) {
		l.width = 0
		return eof
	}
	rune, l.width = int(l.input[l.pos]), 1 //screw utf8
	l.pos += l.width
	return rune
}

func (l *lexer) peek() int {
	rune := l.next()
	l.backup()
	return rune
}

func (l *lexer) backup() {
	l.pos -= l.width
}

func (l *lexer) emit(t tokenType) {
	if t == eofType || t == errorType {
		l.eofd = true
	}
	l.items <- token{t, l.input[l.start:l.pos]}
	l.start = l.pos
}

func (l *lexer) ignore() {
	l.start = l.pos
}

func (l *lexer) accept(valid string) bool {
	if strings.IndexRune(valid, l.next()) >= 0 {
		return true
	}
	l.backup()
	return false
}

func (l *lexer) acceptRun(valid string) {
	for strings.IndexRune(valid, l.next()) >= 0 {
	}
	l.backup()
}

func (l *lexer) acceptLength(n int) {
	for ; n > 0; n-- {
		l.next()
	}
}

func (l *lexer) errorf(format string, args ...interface{}) stateFn {
	l.items <- token{errorType, fmt.Sprintf(format, args...)}
	return nil
}

func (l *lexer) nextToken() token {
	if l.eofd {
		return token{eofType, ""}
	}
	if l.peekBuffer != nil {
		tmp := *l.peekBuffer
		l.peekBuffer = nil
		return tmp
	}

	return <-l.items
}

func (l *lexer) peekToken() token {
	if l.eofd {
		return token{eofType, ""}
	}
	if l.peekBuffer != nil {
		return *l.peekBuffer
	}
	tmp := <-l.items
	l.peekBuffer = &tmp
	return tmp
}

func lex(input string) *lexer {
	l := &lexer{
		input: input,
		state: lexValue,
		items: make(chan token),
		stack: make(tokenStack, 0),
	}<commit_msg>Fixed peek to not check if it errored first<commit_after>package bencode

import (
	"strings"
	"fmt"
	"strconv"
)

type token struct {
	typ tokenType
	val string
}

func (t token) String() string {
	return fmt.Sprintf("[%s]%q", t.typ, t.val)
}

type tokenType int

const (
	eofType = tokenType(iota)
	errorType
	stringType
	intType
	dictStartType
	dictEndType
	listStartType
	listEndType
)

var tokenNames = map[tokenType]string{
	eofType:       "EOF",
	errorType:     "error",
	stringType:    "string",
	intType:       "int",
	dictStartType: "dictStart",
	dictEndType:   "dictEnd",
	listStartType: "listStart",
	listEndType:   "listEnd",
}

func (t tokenType) String() string {
	return tokenNames[t]
}

type tokenStack []tokenType

func (ts *tokenStack) push(t tokenType) {
	*ts = append(tokenStack{t}, *ts...)
}
func (t *tokenStack) pop() tokenType {
	val := (*t)[0]
	*t = (*t)[1:]
	return val
}

const eof = -1

type stateFn func(*lexer) stateFn

type lexer struct {
	input      string
	state      stateFn
	pos        int
	start      int
	width      int
	items      chan token
	peekBuffer *token
	eofd       bool
	stack      tokenStack
}

func (l *lexer) next() (rune int) {
	if l.pos >= len(l.input) {
		l.width = 0
		return eof
	}
	rune, l.width = int(l.input[l.pos]), 1 //screw utf8
	l.pos += l.width
	return rune
}

func (l *lexer) peek() int {
	rune := l.next()
	l.backup()
	return rune
}

func (l *lexer) backup() {
	l.pos -= l.width
}

func (l *lexer) emit(t tokenType) {
	if t == eofType || t == errorType {
		l.eofd = true
	}
	l.items <- token{t, l.input[l.start:l.pos]}
	l.start = l.pos
}

func (l *lexer) ignore() {
	l.start = l.pos
}

func (l *lexer) accept(valid string) bool {
	if strings.IndexRune(valid, l.next()) >= 0 {
		return true
	}
	l.backup()
	return false
}

func (l *lexer) acceptRun(valid string) {
	for strings.IndexRune(valid, l.next()) >= 0 {
	}
	l.backup()
}

func (l *lexer) acceptLength(n int) {
	for ; n > 0; n-- {
		l.next()
	}
}

func (l *lexer) errorf(format string, args ...interface{}) stateFn {
	l.items <- token{errorType, fmt.Sprintf(format, args...)}
	return nil
}

func (l *lexer) nextToken() token {
	if l.peekBuffer != nil {
		tmp := *l.peekBuffer
		l.peekBuffer = nil
		return tmp
	}
	if l.eofd {
		return token{eofType, ""}
	}
	return <-l.items
}

func (l *lexer) peekToken() token {
	if l.peekBuffer != nil {
		return *l.peekBuffer
	}
	if l.eofd {
		return token{eofType, ""}
	}
	tmp := <-l.items
	l.peekBuffer = &tmp
	return tmp
}

func lex(input string) *lexer {
	l := &lexer{
		input: input,
		state: lexValue,
		items: make(chan token),
		stack: make(tokenStack, 0),
	}
{'fraction_non_alphanumeric': 0.15366928530369803, 'fraction_numerical': 0.003832151753209427, 'mean_word_length': 3.1297468354430378, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5018789', 'n_tokens_mistral': 2209, 'n_tokens_neox': 2140, 'n_words': 566}