<commit_before><commit_msg>SVG buffer to store hashes<commit_after>package svg // import "github.com/tdewolff/parse/svg"

import (
	"github.com/tdewolff/parse"
	"github.com/tdewolff/parse/xml"
)

// Token is a single token unit with an attribute value (if given) and hash of the data.
type Token struct {
	xml.TokenType
	Data    []byte
	AttrVal []byte
	Hash    Hash
}

// TokenBuffer is a buffer that allows for token look-ahead.
type TokenBuffer struct {
	l *xml.Lexer

	buf []Token
	pos int
}

// NewTokenBuffer returns a new TokenBuffer.
func NewTokenBuffer(l *xml.Lexer) *TokenBuffer {
	return &TokenBuffer{
		l:   l,
		buf: make([]Token, 0, 8),
	}
}

func (z *TokenBuffer) read(p []Token) int {
	for i := 0; i < len(p); i++ {
		tt, data := z.l.Next()
		if !z.l.IsEOF() {
			data = parse.Copy(data)
		}

		var attrVal []byte
		var hash Hash
		if tt == xml.AttributeToken {
			attrVal = z.l.AttrVal()
			if !z.l.IsEOF() {
				attrVal = parse.Copy(attrVal)
			}
			hash = ToHash(data)
		} else if tt == xml.StartTagToken || tt == xml.EndTagToken {
			hash = ToHash(data)
		}
		p[i] = Token{tt, data, attrVal, hash}
		if tt == xml.ErrorToken {
			return i + 1
		}
	}
	return len(p)
}

// Peek returns the ith element and possibly does an allocation.
// Peeking past an error will panic.
func (z *TokenBuffer) Peek(end int) *Token {
	end += z.pos
	if end >= len(z.buf) {
		c := cap(z.buf)
		d := len(z.buf) - z.pos
		var buf []Token
		if 2*d > c {
			buf = make([]Token, d, 2*c)
		} else {
			buf = z.buf[:d]
		}
		copy(buf, z.buf[z.pos:])

		n := z.read(buf[d:cap(buf)])
		end -= z.pos
		z.pos, z.buf = 0, buf[:d+n]
	}
	return &z.buf[end]
}

// Shift returns the first element and advances position.
func (z *TokenBuffer) Shift() *Token {
	t := z.Peek(0)
	z.pos++
	return t
}

{'fraction_non_alphanumeric': 0.15475520540236354, 'fraction_numerical': 0.004501969611705121, 'mean_word_length': 2.9076923076923076, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2429948', 'n_tokens_mistral': 769, 'n_tokens_neox': 709, 'n_words': 205}