<commit_before>package jo

// Events signal changes in scanning state.
type Event int

const (
	// Nothing of interest, continue scanning.
	None Event = 0

	// Same as None, but specifically for whitespace.
	Space = (1 << iota)

	// Start events.
	ObjectStart = (1 << iota)
	KeyStart
	ArrayStart
	StringStart
	NumberStart
	BoolStart
	NullStart

	// End events.
	ObjectEnd = (1 << iota)
	KeyEnd
	ArrayEnd
	StringEnd
	NumberEnd
	BoolEnd
	NullEnd

	// Syntax error.
	Error = (1 << iota)
)<commit_msg>Add the Start and End bitsets<commit_after>package jo

// Events signal changes in scanning state.
type Event int

const (
	// Nothing of interest, continue scanning.
	None Event = 0

	// Same as None, but specifically for whitespace.
	Space = (1 << iota)

	// Start events.
	ObjectStart = (1 << iota)
	KeyStart
	ArrayStart
	StringStart
	NumberStart
	BoolStart
	NullStart

	// End events.
	ObjectEnd = (1 << iota)
	KeyEnd
	ArrayEnd
	StringEnd
	NumberEnd
	BoolEnd
	NullEnd

	// Start and end bitsets.
	Start = ObjectStart | KeyStart | ArrayStart | StringStart | NumberStart | BoolStart | NullStart
	End   = ObjectEnd | KeyEnd | ArrayEnd | StringEnd | NumberEnd | BoolEnd | NullEnd

	// Syntax error.
	Error = (1 << iota)
)
{'fraction_non_alphanumeric': 0.08949096880131363, 'fraction_numerical': 0.008210180623973728, 'mean_word_length': 3.232638888888889, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2379482', 'n_tokens_mistral': 418, 'n_tokens_neox': 399, 'n_words': 139}