<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package defines constants representing the lexical
// tokens of the Go programming language and basic operations
// on tokens (printing, predicates).
//
package token

import (
	"fmt";
	"strconv";
)


// Token is the set of lexical tokens of the Go programming language.
type Token int

// The list of tokens.
const (
	// Special tokens
	ILLEGAL Token = iota;
	EOF;
	COMMENT;

	// Identifiers and basic type literals
	// (these tokens stand for classes of literals)
	literal_beg;
	IDENT;
	INT;
	FLOAT;
	CHAR;
	STRING;
	literal_end;

	// Operators and delimiters
	operator_beg;
	ADD;
	SUB;
	MUL;
	QUO;
	REM;

	AND;
	OR;
	XOR;
	SHL;
	SHR;
	AND_NOT;

	ADD_ASSIGN;
	SUB_ASSIGN;
	MUL_ASSIGN;
	QUO_ASSIGN;
	REM_ASSIGN;

	AND_ASSIGN;
	OR_ASSIGN;
	XOR_ASSIGN;
	SHL_ASSIGN;
	SHR_ASSIGN;
	AND_NOT_ASSIGN;

	LAND;
	LOR;
	ARROW;
	INC;
	DEC;

	EQL;
	LSS;
	GTR;
	ASSIGN;
	NOT;

	NEQ;
	LEQ;
	GEQ;
	DEFINE;
	ELLIPSIS;

	LPAREN;
	LBRACK;
	LBRACE;
	COMMA;
	PERIOD;

	RPAREN;
	RBRACK;
	RBRACE;
	SEMICOLON;
	COLON;
	operator_end;

	// Keywords
	keyword_beg;
	BREAK;
	CASE;
	CHAN;
	CONST;
	CONTINUE;

	DEFAULT;
	DEFER;
	ELSE;<commit_msg>documentation edits: add comments showing Go syntax for each token. move doc comments below hidden begin constants<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// This package defines constants representing the lexical
// tokens of the Go programming language and basic operations
// on tokens (printing, predicates).
//
package token

import (
	"fmt";
	"strconv";
)


// Token is the set of lexical tokens of the Go programming language.
type Token int

// The list of tokens.
const (
	// Special tokens
	ILLEGAL Token = iota;
	EOF;
	COMMENT;

	literal_beg;
	// Identifiers and basic type literals
	// (these tokens stand for classes of literals)
	IDENT;		// main
	INT;		// 12345
	FLOAT;		// 123.45
	CHAR;		// 'a'
	STRING;		// "abc"
	literal_end;

	operator_beg;
	// Operators and delimiters
	ADD;		// +
	SUB;		// -
	MUL;		// *
	QUO;		// /
	REM;		// %

	AND;		// &
	OR;		// |
	XOR;		// ^
	SHL;		// <<
	SHR;		// >>
	AND_NOT;	// &^

	ADD_ASSIGN;	// +=
	SUB_ASSIGN;	// -=
	MUL_ASSIGN;	// *=
	QUO_ASSIGN;	// /=
	REM_ASSIGN;	// %=

	AND_ASSIGN;	// &=
	OR_ASSIGN;	// |=
	XOR_ASSIGN;	// ^=
	SHL_ASSIGN;	// <<=
	SHR_ASSIGN;	// >>=
	AND_NOT_ASSIGN;	// &^=

	LAND;		// &&
	LOR;		// ||
	ARROW;		// <-
	INC;		// ++
	DEC;		// --

	EQL;		// ==
	LSS;		// <
	GTR;		// >
	ASSIGN;		// =
	NOT;		// !

	NEQ;		// !=
	LEQ;		// <=
	GEQ;		// >=
	DEFINE;		// :=
	ELLIPSIS;	// ...

	LPAREN;		// (
	LBRACK;		// [
	LBRACE;		// {
	COMMA;		// ,
	PERIOD;		// .

	RPAREN;		// )
	RBRACK;		// ]
	RBRACE;		// }
	SEMICOLON;	// ;
	COLON;		// :
	operator_end;

	keyword_beg;
	// Keywords
	BREAK;
	CASE;
	CHAN;
	CONST;
	CONTINUE;

	DEFAULT;
	DEFER;
	ELSE;
{'fraction_non_alphanumeric': 0.1409329829172142, 'fraction_numerical': 0.005913272010512484, 'mean_word_length': 2.4445701357466065, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5868873', 'n_tokens_mistral': 1387, 'n_tokens_neox': 1321, 'n_words': 342}