<commit_before>package tokenizer

// WordProps qualifies a token whose type is one of `TokMayBeWord` or
// `TokWord`.
//
// It holds information regarding the so-called IOB (Inside, Outside,
// Beginning) status of the token, its lemma form (in case of a word),
// its part of speech (in case of a word), etc.
type WordProps struct {
	Word   Token  // The token being qualified.
	IsWord bool   // TokMayBeWord, etc. (false) vs. TokWord (true)
	IOB    byte   // 'B', 'I' or 'O'
	Lemma  string // Lemma form
	PoS    string // Part of Speech
}<commit_msg>Add a `Word` type of token<commit_after>package tokenizer

// Word represents a token whose type is one of `TokMayBeWord` or
// `TokWord`, and qualifies it.
//
// It holds information regarding the so-called IOB (Inside, Outside,
// Beginning) status of the token, its lemma form (in case of a word),
// its part of speech (in case of a word), etc.
type Word struct {
	token TextToken // Actual text and its properties
	iob   byte      // 'B', 'I' or 'O'
	lemma string    // Lemma form
	pos   string    // Part of Speech
}

func (w *Word) Text() string {
	return w.token.text
}

func (w *Word) Begin() int {
	return w.token.begin
}

func (w *Word) End() int {
	return w.token.end
}

func (w *Word) Type() TokenType {
	return w.token.ttype
}

func (w *Word) IOB() byte {
	return w.iob
}

func (w *Word) Lemma() string {
	return w.lemma
}

func (w *Word) POS() string {
	return w.pos
}
{'fraction_non_alphanumeric': 0.12055749128919861, 'fraction_numerical': 0.0, 'mean_word_length': 2.988888888888889, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7345988', 'n_tokens_mistral': 491, 'n_tokens_neox': 478, 'n_words': 206}