<commit_before>package tokenizer

import (
	"bytes"
	"io"
	"unicode"
)

// TextToken represents a piece of text extracted from a larger input.
// It holds information regarding its beginning and ending offsets in
// the input text.  A text token may span the entire input.
type TextToken struct {
	text  string
	begin int
	end   int
}

func (tt *TextToken) Text() string {
	return tt.text
}

func (tt *TextToken) Begin() int {
	return tt.begin
}

func (tt *TextToken) End() int {
	return tt.end
}

// TextTokenIterator helps in retrieving consecutive text tokens from
// an input text.
type TextTokenIterator struct {
	in  []byte
	ct  *TextToken
	idx int
	buf bytes.Buffer
}

func NewTextTokenIterator(input []byte) *TextTokenIterator {
	ti := &TextTokenIterator{}
	ti.in = input
	return ti
}

func NewTextTokenIteratorWithRunningIndex(input []byte, idx int) *TextTokenIterator {
	ti := &TextTokenIterator{}
	ti.in = input
	ti.idx = idx
	return ti
}

func (ti *TextTokenIterator) Item() Token {
	return ti.ct
}

// MoveNext is an iterator method towards implementation of the
// `TokenIterator` interface.<commit_msg>Rename a method for crispness<commit_after>package tokenizer

import (
	"bytes"
	"io"
	"unicode"
)

// TextToken represents a piece of text extracted from a larger input.
// It holds information regarding its beginning and ending offsets in
// the input text.  A text token may span the entire input.
type TextToken struct {
	text  string
	begin int
	end   int
}

func (tt *TextToken) Text() string {
	return tt.text
}

func (tt *TextToken) Begin() int {
	return tt.begin
}

func (tt *TextToken) End() int {
	return tt.end
}

// TextTokenIterator helps in retrieving consecutive text tokens from
// an input text.
type TextTokenIterator struct {
	in  []byte
	ct  *TextToken
	idx int
	buf bytes.Buffer
}

// NewTextTokenIterator creates and initialises a token iterator over
// the given input text.
func NewTextTokenIterator(input []byte) *TextTokenIterator {
	ti := &TextTokenIterator{}
	ti.in = input
	return ti
}

// NewTextTokenIteratorWithOffset creates and initialises a token
// iterator over the given input text.
//
// It treats the given offset - rather than 0 - as the starting index
// from which to track all subsequent indices.
func NewTextTokenIteratorWithOffset(input []byte, n int) *TextTokenIterator {
	ti := &TextTokenIterator{}
	ti.in = input
	ti.idx = n
	return ti
}

func (ti *TextTokenIterator) Item() Token {
	return ti.ct
}

// MoveNext is an iterator method towards implementation of the
// `TokenIterator` interface.
{'fraction_non_alphanumeric': 0.087890625, 'fraction_numerical': 0.000390625, 'mean_word_length': 3.8688212927756656, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1535307', 'n_tokens_mistral': 777, 'n_tokens_neox': 768, 'n_words': 310}