<commit_before>This document specifies the grammar for the Alloy programming language. It's still a work in progress,
some of the language may be missing, or some of the following may be incorrect/invalid or out-dated.
The grammar pretty much maps straight onto the parsers source.

	digit = { "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" } .
	letter = "A" | "a" | ... "Z" | "z" | "_" .
	hex_digit = { "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "a" | "A" | "b" | "B" | "c" | "C" | "d" | "D" | "e" | "E" | "f" | "F" } .
	
	identifier = letter { letter | digit } .
	
	sign = "+" | "-" .
	escaped_char = "\" ( "a" | "b" | "f" | "n" | "r" | "t" | "v" | "\" | "'" | """ ) .
	binaryOp = logOp | relOp | addOp | mulOp .
	logOp = "||" | "&&"
	relOp = "==" | "!=" | "<" | "<=" | ">" | ">=" .
	addOp = "+" | "-" | "|" | "^" .
	mulOp = "*" | "/" | "%" | "<<" | ">>" | "&" .
	unaryOp = "+" | "-" | "!" | "^" | "<" | ">" | "*" | "&" .
	hex_literal = "0" ( "x" | "X" ) hex_digit { hex_digit } .

	Literal = NumberLiteral | StringLiteral | CharacterLiteral .
	NumberLiteral = [sign] digit [ "." { digit } ]	
	StringLiteral = """ { letter } """ . 
	CharacterLiteral = "'"  ( letter | escaped_char ) "'" .
	
	IdentifierList = identifier [ { "," identifier } ]
	ExpressionList = Expression { "," Expression }
	
	Type = TypeName | TypeLit.
	TypeLit = ArrayType | PointerType .
	TypeName = identifier .
	PointerType = "^" BaseType .
	ArrayType = "[" [ Expression ] "]" Type.
	
	FunctionSignature = [ Receiver ] identifier Parameters ":" [ "mut" ] Type .
	Receiver = "(" [ "mut" ] Type identifier ")"
	Parameters = "(" [ parameterList ] ")" .
	ParameterList = ParameterSection { "," ParameterSection } .
	ParameterSection = [ "mut" ] Type identifier .

	StructDecl = "struct" identifier "{" [ FieldDeclList ] "}" .
	FieldDeclList = FieldDecl { FieldDecl } .
	FieldDecl = [ "mut" ] Type IdentifierList ";" .
	FunctionDecl = FunctionSignature ( ";" | Block ) .

	Block = ( "{" [ StatementList ] "}" | "->" Statement ) .
	IfStat = if Expression Block [ "else" Statement ] .
	StatementList = { Statement ";" } .
	MatchStat = "match" Expression "{" { MatchClause "," } "}" . 
	MatchClause = Expression Block ";" . 
	ForStat = "for" Type identifier ":" "(" PrimaryExpr "," PrimaryExpr [ "," PrimaryExpr ] ")" Block .
	
	Declaration = VarDecl | FunctionDecl | StructDecl.
	VarDecl = [ "mut" ] Type identifier [ "=" Expression ] ";" .
	
	Expression = UnaryExpr | Expression binaryOp UnaryExpr .
	UnaryExpr = PrimaryExpr | unaryOp UnaryExpr .

	PrimaryExpr =
		identifier 
		| Literal 
		| "(" Expression ")" 
		| Call 
		| Expression "[" Expression [ ":" Expression ] "]" 
		| Expression "." identifier.
		
	Call = PrimaryExpr "(" [ ExpressionList ] ")" .
	
	Statement = ( StructuredStat | UnstructuredStat ) .
	StructuredStat = Block | IfStat | MatchStat | ForStat .
	UnstructuredStat = Declaration | LeaveStat | IncDecStat | Assignment.
	LeaveStat = ReturnStat | BreakStat | ContinueStat .
	Assignment = PrimaryExpr "=" Expression .
	ReturnStat = "return" [ Expression ] ";" .
	BreakStat = "break" ";"
	ContinueStat = "continue" ";"
	IncDecStat = Expression ( "++" | "--" ) .
	
	
	
	<commit_msg>grammar changes<commit_after>This document specifies the grammar for the Alloy programming language. It's still a work in progress,
some of the language may be missing, or some of the following may be incorrect/invalid or out-dated.
The grammar pretty much maps straight onto the parsers source.

	digit = { "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" };
	letter = "A" | "a" | ... "Z" | "z" | "_";
	hex_digit = { "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" | "a" | "A" | "b" | "B" | "c" | "C" | "d" | "D" | "e" | "E" | "f" | "F" };
	
	identifier = letter { letter | digit };
	
	sign = "+" | "-";
	escaped_char = "\" ( "a" | "b" | "f" | "n" | "r" | "t" | "v" | "\" | "'" | """ );
	binaryOp = logOp | relOp | addOp | mulOp;
	logOp = "||" | "&&"
	relOp = "==" | "!=" | "<" | "<=" | ">" | ">=";
	addOp = "+" | "-" | "|" | "^";
	mulOp = "*" | "/" | "%" | "<<" | ">>" | "&";
	unaryOp = "+" | "-" | "!" | "^" | "<" | ">" | "*" | "&";
	hex_literal = "0" ( "x" | "X" ) hex_digit { hex_digit };

	Literal = NumberLiteral | StringLiteral | CharacterLiteral;
	NumberLiteral = [sign] digit [ "." { digit } ]	
	StringLiteral = """ { letter } """; 
	CharacterLiteral = "'"  ( letter | escaped_char ) "'";
	
	IdentifierList = identifier [ { "," identifier } ]
	ExpressionList = Expression { "," Expression }
	
	Type = TypeName | TypeLit.
	TypeLit = ArrayType | PointerType;
	TypeName = identifier;
	PointerType = "^" BaseType;
	ArrayType = "[" [ Expression ] "]" Type.
	
	FunctionSignature = [ Receiver ] identifier Parameters ":" [ "mut" ] Type;
	Receiver = "(" [ "mut" ] Type identifier ")"
	Parameters = "(" [ parameterList ] ")";
	ParameterList = ParameterSection { "," ParameterSection };
	ParameterSection = [ "mut" ] Type identifier;

	StructDecl = "struct" identifier "{" [ FieldDeclList ] "}";
	FieldDeclList = FieldDecl { FieldDecl };
	FieldDecl = [ "mut" ] Type IdentifierList ";";
	FunctionDecl = FunctionSignature ( ";" | Block );

	Block = ( "{" [ StatementList ] "}" | "->" Statement );
	IfStat = if Expression Block [ "else" Statement ];
	StatementList = { Statement ";" };
	MatchStat = "match" Expression "{" { MatchClause "," } "}"; 
	MatchClause = Expression Block ";"; 
	ForStat = "for" Type identifier ":" "(" PrimaryExpr "," PrimaryExpr [ "," PrimaryExpr ] ")" Block;
	
	Declaration = VarDecl | FunctionDecl | StructDecl.
	VarDecl = [ "mut" ] Type identifier [ "=" Expression ] ";";

	UnaryExpr = unaryOp PrimaryExpr;
	MemberAccess = [ identifier "." ] identifier

	PrimaryExpr =
		Type
		| "(" PrimaryExpr ")"
		| UnaryExpr
		| "^" MemberAccess
		| Literal
		| Call;
		
	Call = PrimaryExpr "(" [ ExpressionList ] ")";
	
	Statement = ( StructuredStat | UnstructuredStat );
	StructuredStat = Block | IfStat | MatchStat | ForStat;
	UnstructuredStat = Declaration | LeaveStat | IncDecStat | Assignment.
	LeaveStat = ReturnStat | BreakStat | ContinueStat;
	Assignment = PrimaryExpr "=" Expression;
	ReturnStat = "return" [ Expression ] ";";
	BreakStat = "break" ";"
	ContinueStat = "continue" ";"
	IncDecStat = Expression ( "++" | "--" );
	
	
	
	
{'fraction_non_alphanumeric': 0.208638826904686, 'fraction_numerical': 0.0066942939113803, 'mean_word_length': 3.2716133424098026, 'pattern_counts': {'":': 5, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2681019', 'n_tokens_mistral': 2251, 'n_tokens_neox': 2126, 'n_words': 519}