<commit_before>package lexer

import (
	"regexp"
	"unicode"

	"github.com/Zac-Garby/pluto/token"
)

var lineEndings = []token.Type{
	token.ID,
	token.String,
	token.Char,
	token.Number,
	token.True,
	token.False,
	token.Null,
	token.Param,
	token.Break,
	token.Next,
	token.Return,
	token.RightParen,
	token.RightSquare,
	token.RightBrace,
}

// Lexer takes a string and returns a stream of tokens
// The stream of tokens is in the form of a function
// which returns the next token.
func Lexer(str string) func() token.Token {
	var (
		index = 0
		col   = 1
		line  = 1
		ch    = make(chan token.Token)
	)

	go func() {
		for {
			if index < len(str) {
				foundSpace := false

				for index < len(str) && (unicode.IsSpace(rune(str[index])) || str[index] == '#') {
					if unicode.IsSpace(rune(str[index])) {
						index++
						col++

						if str[index-1] == '\n' {
							col = 1
							line++
						}

						foundSpace = true<commit_msg>Add GreaterThan to line endings<commit_after>package lexer

import (
	"regexp"
	"unicode"

	"github.com/Zac-Garby/pluto/token"
)

var lineEndings = []token.Type{
	token.ID,
	token.String,
	token.Char,
	token.Number,
	token.True,
	token.False,
	token.Null,
	token.Param,
	token.Break,
	token.Next,
	token.Return,
	token.RightParen,
	token.RightSquare,
	token.RightBrace,
	token.GreaterThan,
}

// Lexer takes a string and returns a stream of tokens
// The stream of tokens is in the form of a function
// which returns the next token.
func Lexer(str string) func() token.Token {
	var (
		index = 0
		col   = 1
		line  = 1
		ch    = make(chan token.Token)
	)

	go func() {
		for {
			if index < len(str) {
				foundSpace := false

				for index < len(str) && (unicode.IsSpace(rune(str[index])) || str[index] == '#') {
					if unicode.IsSpace(rune(str[index])) {
						index++
						col++

						if str[index-1] == '\n' {
							col = 1
							line++
						}

						foundSpace = true
{'fraction_non_alphanumeric': 0.1383975026014568, 'fraction_numerical': 0.005202913631633715, 'mean_word_length': 2.6980769230769233, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7079404', 'n_tokens_mistral': 850, 'n_tokens_neox': 715, 'n_words': 182}