<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/
//
// Copyright <NAME>ர் <<EMAIL>>

package keeri

import (
	"bufio"
	"strings"
	"unicode"
	"unicode/utf8"
)

func isDelim(r rune) bool {
	return unicode.IsSpace(r) || (r == ',') || (r == '<') || (r == '>') ||
		(r == '=') || (r == '"') || (r == '\'') || (r == '(') || (r == ')')
}

func scanTokens(data []byte, atEOF bool) (advance int,
	token []byte, err error) {

	start := 0

	// Scan the token by parsing until a whitespace or a delimiter
	for width, i := 0, start; i < len(data); i += width {
		var r rune
		r, width = utf8.DecodeRune(data[i:])
		if isDelim(r) {
			if i == 0 {
				return width, data[start:width], nil
			}
			return i + width - 1, data[start:i], nil
		}
	}<commit_msg>code comment<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/
//
// Copyright Sankar சங்கர் <<EMAIL>>

package keeri

import (
	"bufio"
	"strings"
	"unicode"
	"unicode/utf8"
)

func isDelim(r rune) bool {
	return unicode.IsSpace(r) || (r == ',') || (r == '<') || (r == '>') ||
		(r == '=') || (r == '"') || (r == '\'') || (r == '(') || (r == ')')
}

func scanTokens(data []byte, atEOF bool) (advance int,
	token []byte, err error) {

	start := 0

	// Scan the token by parsing until a whitespace or a delimiter
	for width, i := 0, start; i < len(data); i += width {
		var r rune
		r, width = utf8.DecodeRune(data[i:])
		if isDelim(r) {
			if i == 0 {
				return width, data[start:width], nil
			}

			// The below (i+width-1) will be valid only until
			// all the delimiters are only 1 character wide
			return i + width - 1, data[start:i], nil
		}
	}
{'fraction_non_alphanumeric': 0.19168831168831169, 'fraction_numerical': 0.011428571428571429, 'mean_word_length': 2.971134020618557, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6911942', 'n_tokens_mistral': 787, 'n_tokens_neox': 724, 'n_words': 244}