<commit_before>// Copyright 2016 <NAME>. All rights reserved.
// Use of this source code is governed by a MIT-style license that can be found
// in the LICENSE file.

package mathcat

import (
	"errors"
	"unicode"
)

const eol rune = -1

// lexer holds the lexer's state while scanning an expression. If any error
// occurs, the scanning stops immediatly and returns the error.
type lexer struct {
	expr   []rune   // the input expression
	ch     rune     // current character
	pos    int      // current character position
	start  int      // current read offset
	tokens []*Token // tokenized lexemes
}

// https://en.wikipedia.org/wiki/Whitespace_character
var whitespaceChars = []rune{
	'\u0009', '\u000A', '\u000B', '\u000C', '\u000D', '\u0020', '\u0085',
	'\u00A0', '\u1680', '\u2000', '\u2001', '\u2002', '\u2003', '\u2004',
	'\u2005', '\u2006', '\u2007', '\u2008', '\u2009', '\u200A', '\u2028',
	'\u2029', '\u202F', '\u205F', '\u3000', '\u180E', '\u200B', '\u200C',
	'\u200D', '\u2060', '\uFEFF',
}

func isIdent(c rune) bool {
	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= 0x80 && unicode.IsLetter(c))
}

func isNumber(c rune) bool {
	return (c >= '0' && c <= '9') || c == '.'
}

func isHex(c rune) bool {
	return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
}

func isBinary(c rune) bool {
	return c == '0' || c == '1'
}

func isOctal(c rune) bool {
	return c >= '0' && c <= '7'
}

func isWhitespace(c rune) bool {
	if c == '\t' || c == ' ' || c == '\r' {
		return true
	}

	for _, v := range whitespaceChars {
		if v == c {
			return true
		}
	}

	return false
}

// Lex starts lexing an expression. We keep reading until EOL is found, which
// we add because we need a padding of 1 to always be able to peek().
//
// Returns the generated tokens and any error found.
func Lex(expr string) ([]*Token, error) {
	l := &lexer{
		expr:  append([]rune(expr), eol), // add eol as padding
		pos:   0,
		start: 0,
	}

	return l.lex()
}
<commit_msg>Move all whitespace characters into slice<commit_after>// Copyright 2016 <NAME>. All rights reserved.
// Use of this source code is governed by a MIT-style license that can be found
// in the LICENSE file.

package mathcat

import (
	"errors"
	"unicode"
)

const eol rune = -1

// lexer holds the lexer's state while scanning an expression. If any error
// occurs, the scanning stops immediatly and returns the error.
type lexer struct {
	expr   []rune   // the input expression
	ch     rune     // current character
	pos    int      // current character position
	start  int      // current read offset
	tokens []*Token // tokenized lexemes
}

// https://en.wikipedia.org/wiki/Whitespace_character
var whitespaceChars = []rune{
	'\t', ' ', '\r', '\n', '\u0009', '\u000A', '\u000B', '\u000C', '\u000D',
	'\u0020', '\u0085', '\u00A0', '\u1680', '\u2000', '\u2001', '\u2002',
	'\u2003', '\u2004', '\u2005', '\u2006', '\u2007', '\u2008', '\u2009',
	'\u200A', '\u2028', '\u2029', '\u202F', '\u205F', '\u3000', '\u180E',
	'\u200B', '\u200C', '\u200D', '\u2060', '\uFEFF',
}

func isIdent(c rune) bool {
	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || (c >= 0x80 && unicode.IsLetter(c))
}

func isNumber(c rune) bool {
	return (c >= '0' && c <= '9') || c == '.'
}

func isHex(c rune) bool {
	return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
}

func isBinary(c rune) bool {
	return c == '0' || c == '1'
}

func isOctal(c rune) bool {
	return c >= '0' && c <= '7'
}

func isWhitespace(c rune) bool {
	for _, v := range whitespaceChars {
		if v == c {
			return true
		}
	}

	return false
}

// Lex starts lexing an expression. We keep reading until EOL is found, which
// we add because we need a padding of 1 to always be able to peek().
//
// Returns the generated tokens and any error found.
func Lex(expr string) ([]*Token, error) {
	l := &lexer{
		expr:  append([]rune(expr), eol), // add eol as padding
		pos:   0,
		start: 0,
	}

	return l.lex()
}

{'fraction_non_alphanumeric': 0.2019422310756972, 'fraction_numerical': 0.06324701195219123, 'mean_word_length': 2.9305283757338554, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1575167', 'n_tokens_mistral': 1715, 'n_tokens_neox': 1504, 'n_words': 516}