<commit_before>package lexer

import (
	"fmt"
	"strings"

	"github.com/sprucehealth/graphql/gqlerrors"
	"github.com/sprucehealth/graphql/language/source"
)

const (
	EOF = iota + 1
	BANG
	DOLLAR
	PAREN_L
	PAREN_R
	SPREAD
	COLON
	EQUALS
	AT
	BRACKET_L
	BRACKET_R
	BRACE_L
	PIPE
	BRACE_R
	NAME
	VARIABLE
	INT
	FLOAT
	STRING
	COMMENT
)

var tokenDescription map[int]string

func init() {
	tokenDescription = make(map[int]string)
	tokenDescription[EOF] = "EOF"
	tokenDescription[BANG] = "!"
	tokenDescription[DOLLAR] = "$"
	tokenDescription[PAREN_L] = "("
	tokenDescription[PAREN_R] = ")"
	tokenDescription[SPREAD] = "..."
	tokenDescription[COLON] = ":"
	tokenDescription[EQUALS] = "="
	tokenDescription[AT] = "@"
	tokenDescription[BRACKET_L] = "["
	tokenDescription[BRACKET_R] = "]"
	tokenDescription[BRACE_L] = "{"
	tokenDescription[PIPE] = "|"
	tokenDescription[BRACE_R] = "}"
	tokenDescription[NAME] = "Name"
	tokenDescription[VARIABLE] = "Variable"
	tokenDescription[INT] = "Int"
	tokenDescription[FLOAT] = "Float"
	tokenDescription[STRING] = "String"
	tokenDescription[COMMENT] = "Comment"
}

type Token struct {
	Kind  int
	Start int
	End   int
	Value string
}

func (t *Token) String() string {
	return tokenDescription[t.Kind]
}

type Lexer struct {
	s   *source.Source
	pos int
}

func New(s *source.Source) *Lexer {
	return &Lexer{s: s}
}

func (l *Lexer) Seek(pos int) {
	l.pos = pos
}

func (l *Lexer) NextToken() (Token, error) {
	token, err := readToken(l.s, l.pos)
	if err != nil {
		return token, err<commit_msg>Add comment for Token struct<commit_after>package lexer

import (
	"fmt"
	"strings"

	"github.com/sprucehealth/graphql/gqlerrors"
	"github.com/sprucehealth/graphql/language/source"
)

const (
	EOF = iota + 1
	BANG
	DOLLAR
	PAREN_L
	PAREN_R
	SPREAD
	COLON
	EQUALS
	AT
	BRACKET_L
	BRACKET_R
	BRACE_L
	PIPE
	BRACE_R
	NAME
	VARIABLE
	INT
	FLOAT
	STRING
	COMMENT
)

var tokenDescription map[int]string

func init() {
	tokenDescription = make(map[int]string)
	tokenDescription[EOF] = "EOF"
	tokenDescription[BANG] = "!"
	tokenDescription[DOLLAR] = "$"
	tokenDescription[PAREN_L] = "("
	tokenDescription[PAREN_R] = ")"
	tokenDescription[SPREAD] = "..."
	tokenDescription[COLON] = ":"
	tokenDescription[EQUALS] = "="
	tokenDescription[AT] = "@"
	tokenDescription[BRACKET_L] = "["
	tokenDescription[BRACKET_R] = "]"
	tokenDescription[BRACE_L] = "{"
	tokenDescription[PIPE] = "|"
	tokenDescription[BRACE_R] = "}"
	tokenDescription[NAME] = "Name"
	tokenDescription[VARIABLE] = "Variable"
	tokenDescription[INT] = "Int"
	tokenDescription[FLOAT] = "Float"
	tokenDescription[STRING] = "String"
	tokenDescription[COMMENT] = "Comment"
}

// Token is a representation of a lexed Token. Value only appears for non-punctuation
// tokens: NAME, INT, FLOAT, and STRING.
type Token struct {
	Kind  int
	Start int
	End   int
	Value string
}

func (t *Token) String() string {
	return tokenDescription[t.Kind]
}

type Lexer struct {
	s   *source.Source
	pos int
}

func New(s *source.Source) *Lexer {
	return &Lexer{s: s}
}

func (l *Lexer) Seek(pos int) {
	l.pos = pos
}

func (l *Lexer) NextToken() (Token, error) {
	token, err := readToken(l.s, l.pos)
	if err != nil {
		return token, err
{'fraction_non_alphanumeric': 0.13439849624060152, 'fraction_numerical': 0.0006265664160401002, 'mean_word_length': 4.631393298059964, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2814241', 'n_tokens_mistral': 1288, 'n_tokens_neox': 1253, 'n_words': 260}