<commit_before>package algebra

import (
	"fmt"
	"unicode"
)

const (
	NumberToken         = "Number"
	SkipToken           = "Skip"
	LeftBracketToken    = "LeftBracket"
	RightBracketToken   = "RightBracket"
	AdditionToken       = "Addition"
	SubtractionToken    = "Subtraction"
	MultiplicationToken = "Multiplication"
	DivisionToken       = "Division"
)

type Token struct {
	Name     string
	Contents string
}

func (t Token) Type(n string) bool {
	if n == t.Name {
		return true
	}

	return false
}

type Lexer struct {
	source   string
	location int
}

func (lex *Lexer) Lex() []Token {
	return lex.lex()
}

func (lex *Lexer) lex() []Token {
	var stream []Token
	for !lex.End() {
		c := lex.source[lex.location]

		if lex.isNumber(c) {
			stream = append(stream, lex.number())
			continue
		}

		if lex.isBrackets(c) {
			stream = append(stream, lex.brackets())
			continue
		}

		if lex.isAddition(c) {
			stream = append(stream, lex.addition())
			continue
		}

		if lex.isSubtraction(c) {
			stream = append(stream, lex.subtraction())
			continue
		}

		if lex.isMultiplication(c) {
			stream = append(stream, lex.multiplication())
			continue
		}

		if lex.isDivision(c) {
			stream = append(stream, lex.division())
			continue
		}

		if lex.isWhitespace(c) {
			stream = append(stream, lex.whitespace())
			continue
		}

		fmt.Println("Invalid token!")
		lex.Next()
	}

	return stream
}

func (lex *Lexer) addition() Token {
	lex.Next()

	return Token{AdditionToken, "+"}
}

func (lex *Lexer) subtraction() Token {
	lex.Next()

	return Token{SubtractionToken, "-"}
}

func (lex *Lexer) multiplication() Token {
	lex.Next()

	return Token{MultiplicationToken, "*"}
}

func (lex *Lexer) division() Token {
	lex.Next()

	return Token{DivisionToken, "/"}
}

func (lex *Lexer) whitespace() Token {<commit_msg>lexer: trying to improve readability of lex func<commit_after>package algebra

import (
	"fmt"
	"unicode"
)

const (
	NumberToken         = "Number"
	SkipToken           = "Skip"
	LeftBracketToken    = "LeftBracket"
	RightBracketToken   = "RightBracket"
	AdditionToken       = "Addition"
	SubtractionToken    = "Subtraction"
	MultiplicationToken = "Multiplication"
	DivisionToken       = "Division"
)

type Token struct {
	Name     string
	Contents string
}

func (t Token) Type(n string) bool {
	if n == t.Name {
		return true
	}

	return false
}

type Lexer struct {
	source   string
	location int
}

func (lex *Lexer) Lex() []Token {
	return lex.lex()
}

func (lex *Lexer) lex() []Token {
	var stream []Token
	for !lex.End() {
		c := lex.source[lex.location]

		if lex.isNumber(c) {
			stream = append(stream, lex.number())
		} else if lex.isBrackets(c) {
			stream = append(stream, lex.brackets())
		} else if lex.isAddition(c) {
			stream = append(stream, lex.addition())
		} else if lex.isSubtraction(c) {
			stream = append(stream, lex.subtraction())
		} else if lex.isMultiplication(c) {
			stream = append(stream, lex.multiplication())
		} else if lex.isDivision(c) {
			stream = append(stream, lex.division())
		} else if lex.isWhitespace(c) {
			stream = append(stream, lex.whitespace())
		} else {
			fmt.Println("Invalid token!")
			return stream
		}

	}

	return stream
}

func (lex *Lexer) addition() Token {
	lex.Next()

	return Token{AdditionToken, "+"}
}

func (lex *Lexer) subtraction() Token {
	lex.Next()

	return Token{SubtractionToken, "-"}
}

func (lex *Lexer) multiplication() Token {
	lex.Next()

	return Token{MultiplicationToken, "*"}
}

func (lex *Lexer) division() Token {
	lex.Next()

	return Token{DivisionToken, "/"}
}

func (lex *Lexer) whitespace() Token {
{'fraction_non_alphanumeric': 0.13766305856230918, 'fraction_numerical': 0.0, 'mean_word_length': 3.4438964241676944, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2506652', 'n_tokens_mistral': 1435, 'n_tokens_neox': 1262, 'n_words': 312}