<commit_before>package retrie

import (
	"github.com/philharnish/forge/src/data/graph/bloom/mask"
	"github.com/philharnish/forge/src/data/graph/bloom/node"
)

type reTrieItems struct {
	acceptor   node.NodeAcceptor
	root       *reTrieNode
	index      int
	runeIndex  int
	runeOffset rune
}

const DOT_PREFIX = "."
const OPTIONAL_PREFIX = "?"

func newTrieItems(acceptor node.NodeAcceptor, root *reTrieNode) node.NodeItems {
	optionalLink := root.optionalLink()
	if optionalLink != nil {
		// The "?" operator has zero length so iterate that node directly.
		return optionalLink.node.Items(acceptor)
	}
	return &reTrieItems{
		acceptor: acceptor,
		root:     root,
	}
}

func (items *reTrieItems) HasNext() bool {
	return items.index < len(items.root.links)
}

func (items *reTrieItems) Next() (string, node.NodeIterator) {
	link := items.root.links[items.index]
	if link.prefix == OPTIONAL_PREFIX {
		panic("Optional iteration should happen before reaching this point.")
	} else if link.prefix == DOT_PREFIX {
		prefix := string(mask.ALPHABET[items.runeIndex])
		items.runeIndex++
		if items.runeIndex >= mask.SIZE {
			items.index++
		}
		return prefix, link.node
	} else if link.runes == nil {
		items.index++
		return link.prefix, link.node
	}
	prefix := string(link.runes[items.runeIndex] + items.runeOffset)
	maxOffset := link.runes[items.runeIndex+1] - link.runes[items.runeIndex]
	items.runeOffset++
	if items.runeOffset > maxOffset {
		items.runeOffset = 0
		items.runeIndex += 2
	}
	if items.runeIndex >= len(link.runes) {
		items.runeIndex = 0
		items.index++
	}
	return prefix, link.node
}<commit_msg>Prefer to return prefix if present.<commit_after>package retrie

import (
	"github.com/philharnish/forge/src/data/graph/bloom/mask"
	"github.com/philharnish/forge/src/data/graph/bloom/node"
)

type reTrieItems struct {
	acceptor   node.NodeAcceptor
	root       *reTrieNode
	index      int
	runeIndex  int
	runeOffset rune
}

const DOT_PREFIX = "."
const OPTIONAL_PREFIX = "?"

func newTrieItems(acceptor node.NodeAcceptor, root *reTrieNode) node.NodeItems {
	optionalLink := root.optionalLink()
	if optionalLink != nil {
		// The "?" operator has zero length so iterate that node directly.
		return optionalLink.node.Items(acceptor)
	}
	return &reTrieItems{
		acceptor: acceptor,
		root:     root,
	}
}

func (items *reTrieItems) HasNext() bool {
	return items.index < len(items.root.links)
}

func (items *reTrieItems) Next() (string, node.NodeIterator) {
	link := items.root.links[items.index]
	if link.prefix == OPTIONAL_PREFIX {
		panic("Optional iteration should happen before reaching this point.")
	} else if link.prefix == DOT_PREFIX {
		prefix := string(mask.ALPHABET[items.runeIndex])
		items.runeIndex++
		if items.runeIndex >= mask.SIZE {
			items.index++
		}
		return prefix, link.node
	} else if link.prefix != "" {
		items.index++
		return link.prefix, link.node
	}
	prefix := string(link.runes[items.runeIndex] + items.runeOffset)
	maxOffset := link.runes[items.runeIndex+1] - link.runes[items.runeIndex]
	items.runeOffset++
	if items.runeOffset > maxOffset {
		items.runeOffset = 0
		items.runeIndex += 2
	}
	if items.runeIndex >= len(link.runes) {
		items.runeIndex = 0
		items.index++
	}
	return prefix, link.node
}
{'fraction_non_alphanumeric': 0.12353666050523722, 'fraction_numerical': 0.0024645717806531116, 'mean_word_length': 4.01854714064915, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4429760', 'n_tokens_mistral': 1218, 'n_tokens_neox': 1160, 'n_words': 250}