<commit_before>package lexer

import (
	"fmt"
	"strings"
	"unicode"
	"unicode/utf8"
)

const (
	eof                        rune   = 0
	leftParenthesis            string = "("
	rightParenthesis                  = ")"
	singleQuotationMark               = "'"
	doubleQuotationMark               = "\""
	escapedSingleQuotationMark        = "\\'"
	escapedDoubleQuotationMark        = "\\\""
	comma                             = ","
	regexEqual                        = "~="
	stringEqual                       = "=="
	not                               = "!"
	and                               = "&&"
	or                                = "||"
)

const (
	ErrorClosingSingleQuotationMarkIsMissing = "Missing closing single quotation mark"
	ErrorClosingDoubleQuotationMarkIsMissing = "Missing closing double quotation mark"
	ErrorClosingParenthesisMissing           = "Missing closing parenthesis"
	ErrorUnexpectedToken                     = "Unexpected token %c"
)

// TokenType represents a type of recognized token
type TokenType int

const (
	TokenError TokenType = iota
	TokenEOF

	TokenLeftParenthesis
	TokenRightParenthesis
	TokenComma
	TokenRegexEqual
	TokenStringEqual
	TokenNot
	TokenAnd
	TokenOr
	TokenSingleQuotedStringLiteral
	TokenDoubleQuotedStringLiteral
	TokenSha1
)

type Token struct {
	Type  TokenType
	Value string
}

type LexFn func(*Lexer) LexFn

type Lexer struct {
	Input                string
	Tokens               []Token
	State                LexFn
	Error                error
	TokenStart           int
	Position             int
	OpenParenthesisCount int
}

func New(input string) *Lexer {
	return &Lexer{Input: input, State: LexBegin}
}

func (lexer *Lexer) Lex() error {
	for {
		if lexer.State == nil {
			break
		}

		lexer.State = (lexer.State)(lexer)
	}

	return lexer.Error
}

func (lexer *Lexer) Emit(tokenType TokenType) {
	lexer.Tokens = append(lexer.Tokens, Token{Type: tokenType, Value: lexer.Input[lexer.TokenStart:lexer.Position]})
}

func (lexer *Lexer) RemainingInput() string {
	return lexer.Input[lexer.Position:]
}

func (lexer *Lexer) Read() rune {
	lexer.Position++

	if lexer.Position >= utf8.RuneCountInString(lexer.Input) {
		return eof
	}

	return rune(lexer.Input[lexer.Position])
}

func (lexer *Lexer) EatWhitespaces() rune {
	var ch rune = eof
	for {
		ch = lexer.Read()

		if !unicode.IsSpace(ch) {
			lexer.Position--
			break
		}

		if ch == eof {
			break
		}
	}

	return ch
}

func LexComma(lexer *Lexer) LexFn {
	lexer.TokenStart = lexer.Position
	lexer.Position++
	lexer.Emit(TokenComma)

	if lexer.IsEOF() {
		lexer.TokenStart = lexer.Position
		lexer.Emit(TokenEOF)
		return lexer.Errorf(ErrorUnexpectedToken)
	}

	return LexBegin
}

func LexLeftParenthesis(lexer *Lexer) LexFn {
	lexer.TokenStart = lexer.Position
	lexer.Position++
	lexer.Emit(TokenLeftParenthesis)
	lexer.OpenParenthesisCount++

	if lexer.IsEOF() {
		lexer.TokenStart = lexer.Position
		lexer.Emit(TokenEOF)
		return lexer.Errorf(ErrorClosingParenthesisMissing)
	}

	return LexBegin
}

func LexRightParenthesis(lexer *Lexer) LexFn {
	lexer.TokenStart = lexer.Position
	lexer.Position++
	lexer.Emit(TokenRightParenthesis)
	lexer.OpenParenthesisCount--

	if lexer.IsEOF() {
		lexer.TokenStart = lexer.Position
		lexer.Emit(TokenEOF)
		return nil
	}

	return LexBegin
}

func (lexer *Lexer) IsEOF() bool {
	return lexer.Position >= utf8.RuneCountInString(lexer.Input)
}

func (lexer *Lexer) Errorf(err string) LexFn {
	lexer.Error = fmt.Errorf("%s at position: %d", err, lexer.Position)
	return nil
}

func LexSingleQuotedString(lexer *Lexer) LexFn {
	lexer.Position++
	lexer.TokenStart = lexer.Position
	for {
		if lexer.IsEOF() {
			lexer.TokenStart = lexer.Position
			lexer.Emit(TokenEOF)
			return lexer.Errorf(ErrorClosingSingleQuotationMarkIsMissing)
		}

		switch {
		case strings.HasPrefix(lexer.RemainingInput(), escapedSingleQuotationMark):
			lexer.Position += len(escapedSingleQuotationMark)
		case strings.HasPrefix(lexer.RemainingInput(), singleQuotationMark):
			lexer.Emit(TokenSingleQuotedStringLiteral)
			lexer.Position++
			return LexBegin
		default:
			lexer.Position++
		}
	}
}

func LexDoubleQuotedString(lexer *Lexer) LexFn {
	lexer.Position++
	lexer.TokenStart = lexer.Position
	for {
		if lexer.IsEOF() {
			lexer.TokenStart = lexer.Position
			lexer.Emit(TokenEOF)
			return lexer.Errorf(ErrorClosingDoubleQuotationMarkIsMissing)
		}

		switch {
		case strings.HasPrefix(lexer.RemainingInput(), escapedDoubleQuotationMark):
			lexer.Position += len(escapedDoubleQuotationMark)
		case strings.HasPrefix(lexer.RemainingInput(), doubleQuotationMark):
			lexer.Emit(TokenDoubleQuotedStringLiteral)
			lexer.Position++
			return LexBegin
		default:
			lexer.Position++
		}
	}
}

func LexBegin(lexer *Lexer) LexFn {
	lexer.EatWhitespaces()

	if lexer.IsEOF() {
		lexer.TokenStart = lexer.Position
		lexer.Emit(TokenEOF)

		if lexer.OpenParenthesisCount > 0 {
			return lexer.Errorf(ErrorClosingParenthesisMissing)
		}

		return nil
	}

	remainingInput := lexer.RemainingInput()

	switch {
	case strings.HasPrefix(remainingInput, leftParenthesis):
		return LexLeftParenthesis
	case strings.HasPrefix(remainingInput, rightParenthesis):
		return LexRightParenthesis
	case strings.HasPrefix(remainingInput, comma):
		return LexComma
	case strings.HasPrefix(remainingInput, singleQuotationMark):
		return LexSingleQuotedString
	case strings.HasPrefix(remainingInput, doubleQuotationMark):
		return LexDoubleQuotedString
	default:
		return lexer.Errorf(fmt.Sprintf(ErrorUnexpectedToken, remainingInput[0]))
	}
}
<commit_msg>added string equal and regex equal operators<commit_after>package lexer

import (
	"fmt"
	"strings"
	"unicode"
	"unicode/utf8"
)

const (
	eof                        rune   = 0
	leftParenthesis            string = "("
	rightParenthesis                  = ")"
	singleQuotationMark               = "'"
	doubleQuotationMark               = "\""
	escapedSingleQuotationMark        = "\\'"
	escapedDoubleQuotationMark        = "\\\""
	comma                             = ","
	regexEqual                        = "~="
	stringEqual                       = "=="
	not                               = "!"
	and                               = "&&"
	or                                = "||"
)

const (
	ErrorClosingSingleQuotationMarkIsMissing = "Missing closing single quotation mark"
	ErrorClosingDoubleQuotationMarkIsMissing = "Missing closing double quotation mark"
	ErrorClosingParenthesisMissing           = "Missing closing parenthesis"
	ErrorUnexpectedToken                     = "Unexpected token %c"
)

// TokenType represents a type of recognized token
type TokenType int

const (
	TokenError TokenType = iota
	TokenEOF

	TokenLeftParenthesis
	TokenRightParenthesis
	TokenComma
	TokenRegexEqual
	TokenStringEqual
	TokenNot
	TokenAnd
	TokenOr
	TokenSingleQuotedStringLiteral
	TokenDoubleQuotedStringLiteral
	TokenSha1
)

type Token struct {
	Type  TokenType
	Value string
}

type LexFn func(*Lexer) LexFn

type Lexer struct {
	Input                string
	Tokens               []Token
	State                LexFn
	Error                error
	TokenStart           int
	Position             int
	OpenParenthesisCount int
}

func New(input string) *Lexer {
	return &Lexer{Input: input, State: LexBegin}
}

func (lexer *Lexer) Lex() error {
	for {
		if lexer.State == nil {
			break
		}

		lexer.State = (lexer.State)(lexer)
	}

	return lexer.Error
}

func (lexer *Lexer) Emit(tokenType TokenType) {
	lexer.Tokens = append(lexer.Tokens, Token{Type: tokenType, Value: lexer.Input[lexer.TokenStart:lexer.Position]})
}

func (lexer *Lexer) RemainingInput() string {
	return lexer.Input[lexer.Position:]
}

func (lexer *Lexer) Read() rune {
	if lexer.Position++; lexer.Position >= utf8.RuneCountInString(lexer.Input) {
		return eof
	}

	return rune(lexer.Input[lexer.Position-1])
}

func (lexer *Lexer) EatWhitespaces() rune {
	var ch rune

	for {
		ch = lexer.Read()

		if !unicode.IsSpace(ch) {
			lexer.Position--
			break
		}

		if ch == eof {
			break
		}
	}

	return ch
}

func LexComma(lexer *Lexer) LexFn {
	lexer.TokenStart = lexer.Position
	lexer.Position++
	lexer.Emit(TokenComma)

	if lexer.IsEOF() {
		lexer.TokenStart = lexer.Position
		lexer.Emit(TokenEOF)
		return nil
	}

	return LexBegin
}

func LexLeftParenthesis(lexer *Lexer) LexFn {
	lexer.TokenStart = lexer.Position
	lexer.Position++
	lexer.Emit(TokenLeftParenthesis)
	lexer.OpenParenthesisCount++

	if lexer.IsEOF() {
		lexer.TokenStart = lexer.Position
		lexer.Emit(TokenEOF)
		return lexer.Errorf(ErrorClosingParenthesisMissing)
	}

	return LexBegin
}

func LexRightParenthesis(lexer *Lexer) LexFn {
	lexer.TokenStart = lexer.Position
	lexer.Position++
	lexer.Emit(TokenRightParenthesis)
	lexer.OpenParenthesisCount--

	if lexer.IsEOF() {
		lexer.TokenStart = lexer.Position
		lexer.Emit(TokenEOF)
		return nil
	}

	return LexBegin
}

func (lexer *Lexer) IsEOF() bool {
	return lexer.Position >= utf8.RuneCountInString(lexer.Input)
}

func (lexer *Lexer) Errorf(err string) LexFn {
	lexer.Error = fmt.Errorf("%s at position: %d", err, lexer.Position)
	return nil
}

func LexSingleQuotedString(lexer *Lexer) LexFn {
	lexer.Position++
	lexer.TokenStart = lexer.Position
	for {
		if lexer.IsEOF() {
			lexer.TokenStart = lexer.Position
			lexer.Emit(TokenEOF)
			return lexer.Errorf(ErrorClosingSingleQuotationMarkIsMissing)
		}

		switch {
		case strings.HasPrefix(lexer.RemainingInput(), escapedSingleQuotationMark):
			lexer.Position += len(escapedSingleQuotationMark)
		case strings.HasPrefix(lexer.RemainingInput(), singleQuotationMark):
			lexer.Emit(TokenSingleQuotedStringLiteral)
			lexer.Position++
			return LexBegin
		default:
			lexer.Position++
		}
	}
}

func LexDoubleQuotedString(lexer *Lexer) LexFn {
	lexer.Position++
	lexer.TokenStart = lexer.Position
	for {
		if lexer.IsEOF() {
			lexer.TokenStart = lexer.Position
			lexer.Emit(TokenEOF)
			return lexer.Errorf(ErrorClosingDoubleQuotationMarkIsMissing)
		}

		switch {
		case strings.HasPrefix(lexer.RemainingInput(), escapedDoubleQuotationMark):
			lexer.Position += len(escapedDoubleQuotationMark)
		case strings.HasPrefix(lexer.RemainingInput(), doubleQuotationMark):
			lexer.Emit(TokenDoubleQuotedStringLiteral)
			lexer.Position++
			return LexBegin
		default:
			lexer.Position++
		}
	}
}

func LexStringEqual(lexer *Lexer) LexFn {
	lexer.TokenStart = lexer.Position
	lexer.Position += len(stringEqual)
	lexer.Emit(TokenStringEqual)

	if lexer.IsEOF() {
		lexer.TokenStart = lexer.Position
		lexer.Emit(TokenEOF)
		return nil
	}

	return LexBegin
}

func LexRegexEqual(lexer *Lexer) LexFn {
	lexer.TokenStart = lexer.Position
	lexer.Position += len(regexEqual)
	lexer.Emit(TokenRegexEqual)

	if lexer.IsEOF() {
		lexer.TokenStart = lexer.Position
		lexer.Emit(TokenEOF)
		return nil
	}

	return LexBegin
}

func LexBegin(lexer *Lexer) LexFn {
	lexer.EatWhitespaces()

	if lexer.IsEOF() {
		lexer.TokenStart = lexer.Position
		lexer.Emit(TokenEOF)

		if lexer.OpenParenthesisCount > 0 {
			return lexer.Errorf(ErrorClosingParenthesisMissing)
		}

		return nil
	}

	remainingInput := lexer.RemainingInput()

	switch {
	case strings.HasPrefix(remainingInput, leftParenthesis):
		return LexLeftParenthesis
	case strings.HasPrefix(remainingInput, rightParenthesis):
		return LexRightParenthesis
	case strings.HasPrefix(remainingInput, comma):
		return LexComma
	case strings.HasPrefix(remainingInput, singleQuotationMark):
		return LexSingleQuotedString
	case strings.HasPrefix(remainingInput, doubleQuotationMark):
		return LexDoubleQuotedString
	case strings.HasPrefix(remainingInput, stringEqual):
		return LexStringEqual
	case strings.HasPrefix(remainingInput, regexEqual):
		return LexRegexEqual
	default:
		return lexer.Errorf(fmt.Sprintf(ErrorUnexpectedToken, remainingInput[0]))
	}
}

{'fraction_non_alphanumeric': 0.09581955397269566, 'fraction_numerical': 0.0012719409819384382, 'mean_word_length': 3.4438583270535044, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6090359', 'n_tokens_mistral': 4308, 'n_tokens_neox': 3953, 'n_words': 771}