<commit_before>header {
package com.xruby.compiler.parser;
}

/** 
 * Copyright (c) 2005-2006 <NAME>. All rights reserved.
 */

//----------------------------------------------------------------------------
// The Ruby parser. Do not use this one, use RubyParser instead!
//----------------------------------------------------------------------------
class RubyParserBase extends Parser;
options {
	k = 1;                           // Will add prediction manually if we need k > 1.
	exportVocab=Ruby;
	codeGenMakeSwitchThreshold = 2;
	codeGenBitsetTestThreshold = 999;
	defaultErrorHandler = false;
	buildAST = true;
	//analyzerDebug = true;
}

tokens {
	// "imaginary" tokens
	RPAREN_IN_METHOD_DEFINATION;
	BODY;
	CALL;
	ARG;
	COMPSTMT;
	SYMBOL;
	BLOCK;
	MULTIPLE_ASSIGN;
	MULTIPLE_ASSIGN_WITH_EXTRA_COMMA;
	MRHS;
	NESTED_LHS;
	SINGLETON_METHOD;
	STRING;
}

{
	private int can_be_command_ = 0;

	protected void enterScope()	{assert(false);}
	protected void enterBlockScope()	{assert(false);}
	protected void leaveScope()	{assert(false);}
	protected void addVariable(Token id)	{assert(false);}
	protected void tellLexerWeHaveFinishedParsingMethodparameters()	{assert(false);}
	protected void tellLexerWeHaveFinishedParsingSymbol()	{assert(false);}
	protected void tellLexerWeHaveFinishedParsingStringExpressionSubstituation()	{assert(false);}
	protected void tellLexerWeHaveFinishedParsingRegexExpressionSubstituation()	{assert(false);}
	protected void tellLexerWeHaveFinishedParsingHeredocExpressionSubstituation()	{assert(false);}

	boolean isNotAssign(int token_type) {
		switch (token_type) {
			case ASSIGN:
			case ASSIGN_WITH_NO_LEADING_SPACE:
			case PLUS_ASSIGN:
			case MINUS_ASSIGN:
			case STAR_ASSIGN:
			case DIV_ASSIGN:
			case MOD_ASSIGN:
			case POWER_ASSIGN:
			case BAND_ASSIGN:
			case BXOR_ASSIGN:
			case BOR_ASSIGN:
			case LEFT_SHIFT_ASSIGN:
			case RIGHT_SHIFT_ASSIGN:
			case LOGICAL_AND_ASSIGN:
			case LOGICAL_OR_ASSIGN:
				return false;
			default:
				return true;
		}
	}
}

program
		:	(compoundStatement)? EOF!
		;

compoundStatement
{boolean empty = true;}
		:	(terminal!	(statements	{empty = false;})?
			|	statements	{empty = false;})
			{if (!empty)	#compoundStatement = #(#[COMPSTMT, "COMPSTMT"], #compoundStatement);}
		;

statements
		:	statement
			(
				terminal
				{
					if (	(EOF == LA(1))	||	//script end
						(RCURLY == LA(1))	||	(LITERAL_end == LA(1))	||	//block end
						(RPAREN == LA(1))	||
						(LITERAL_else == LA(1)) || (LITERAL_elsif == LA(1))	||
						(LITERAL_rescue == LA(1)) || (LITERAL_ensure == LA(1))	||
						(LITERAL_when == LA(1))
					)
					break;
				}
				statement
			)*
		;

terminal
		:	SEMI!
		|	LINE_BREAK!
		;

statement
		:	body:statementWithoutModifier
									(!IF_MODIFIER^		if_condition:expression	{#statement = #(#[LITERAL_if, "if"], if_condition, #(#[COMPSTMT, "COMPSTMT"], #body));}
									|!UNLESS_MODIFIER^	unless_condition:expression	{#statement = #(#[LITERAL_unless, "unless"], unless_condition, #(#[COMPSTMT, "COMPSTMT"], #body));}
									|!WHILE_MODIFIER^	while_condition:expression	{#statement = #(#[LITERAL_while, "while"], while_condition, #(#[COMPSTMT, "COMPSTMT"], #body));}
									|!UNTIL_MODIFIER^	until_condition:expression	{#statement = #(#[LITERAL_until, "until"], until_condition, #(#[COMPSTMT, "COMPSTMT"], #body));}
									|!RESCUE_MODIFIER^	rescue_condition:expression	{#statement = #(#[LITERAL_rescue, "rescue"], rescue_condition, #(#[COMPSTMT, "COMPSTMT"], #body));}
									)*
		;

statementWithoutModifier
{
	boolean is_parallel_assignment = false;
	boolean has_extra_comma = false;
}
		:	(aliasStatement
			|	undefStatement
			|	keyword_BEGIN 	LCURLY_BLOCK (compoundStatement)? RCURLY
			|	keyword_END 	LCURLY_BLOCK (compoundStatement)? RCURLY
			|	expression
				(options{greedy=true;}:
					COMMA!
					{if (ASSIGN == LA(1) || ASSIGN_WITH_NO_LEADING_SPACE == LA(1)) {has_extra_comma=true;break;}}
					(seen_star:REST_ARG_PREFIX)?
					(mlhs_item)?	{if (null != seen_star) break;}
					{is_parallel_assignment = true;}
				)*
				(	(ASSIGN!|ASSIGN_WITH_NO_LEADING_SPACE!)
					(options{greedy=true;}:LINE_BREAK!)?
					mrhs
					{is_parallel_assignment = true;}
				)?
			|	REST_ARG_PREFIX
				mlhs_item
				(ASSIGN!|ASSIGN_WITH_NO_LEADING_SPACE!)
				(options{greedy=true;}:LINE_BREAK!)?
				mrhs
				{is_parallel_assignment = true;}
			)
			{	
				if (is_parallel_assignment && !has_extra_comma) {
					#statementWithoutModifier = #(#[MULTIPLE_ASSIGN, "MULTIPLE_ASSIGN"], #statementWithoutModifier);
				} else if (is_parallel_assignment && has_extra_comma) {
					#statementWithoutModifier = #(#[MULTIPLE_ASSIGN_WITH_EXTRA_COMMA, "MULTIPLE_ASSIGN_WITH_EXTRA_COMMA"], #statementWithoutModifier);
				}
			}
		;

//LPAREN can start primaryExpression, have to use syntactic predicate here.
mlhs_item
		:	(LPAREN!	dotColonOrArrayAccess	COMMA)=>
			LPAREN!
			dotColonOrArrayAccess
			(COMMA!	(seen_star:REST_ARG_PREFIX)?	dotColonOrArrayAccess	{if (null != seen_star) break;})+
			RPAREN!
			{#mlhs_item = #(#[NESTED_LHS, "NESTED_LHS"], #mlhs_item);}
		|	dotColonOrArrayAccess
		;

mrhs
		:	(expression	(COMMA! {if (ASSIGN == LA(1)||RBRACK == LA(1)) break;}	(seen_star:REST_ARG_PREFIX)?	expression	{if (null != seen_star) break;})*
			|	REST_ARG_PREFIX	expression
			)
			{#mrhs = #(#[MRHS, "MRHS"], #mrhs);}
		;

protected
undef_parameter
		:	IDENTIFIER	(ASSIGN_WITH_NO_LEADING_SPACE)?
		|	CONSTANT	(ASSIGN_WITH_NO_LEADING_SPACE)?
		|	FUNCTION	(ASSIGN_WITH_NO_LEADING_SPACE)?
		|	symbol
		|	keywordAsMethodName
		|	operatorAsMethodname
		;

undefStatement
		:	"undef"^	(options{greedy=true;}:LINE_BREAK!)?
			undef_parameter	(COMMA!	undef_parameter)*
		;

aliasStatement
		:	"alias"^	(options{greedy=true;}:LINE_BREAK!)?
			(GLOBAL_VARIABLE 	(LINE_BREAK!)?	GLOBAL_VARIABLE 
			|undef_parameter	(LINE_BREAK!)?	undef_parameter
			)
		;

//LPAREN can start primaryExpression, have to use syntactic predicate here.
block_var
		:	(LPAREN!	dotColonOrArrayAccess	COMMA!)=>	LPAREN	dotColonOrArrayAccess	(COMMA!	{if (RPAREN == LA(1)) break;}	(seen_star:REST_ARG_PREFIX)?	dotColonOrArrayAccess	{if (null != seen_star) break;})*		RPAREN!
		|	REST_ARG_PREFIX	(dotColonOrArrayAccess)?
		|	dotColonOrArrayAccess
		;

//TODO should be mlhs
block_vars
		:	block_var		(COMMA!	{if (BOR == LA(1) || LITERAL_in == LA(1)) break;}	block_var	)*
		;

codeBlock
		:	("do"!	(options{greedy=true;}:LINE_BREAK!)?		{enterBlockScope();}
				blockContent
				"end"!			{leaveScope();}
			|	LCURLY_BLOCK!	{enterBlockScope();}
				blockContent
				RCURLY!			{leaveScope();}
			)
			{#codeBlock = #(#[BLOCK, "BLOCK"], #codeBlock);}
		;

protected
blockContent
		:	(BOR	(block_vars)?	BOR!
			|LOGICAL_OR
			)?	(compoundStatement)?
		;

symbol
		:	COLON_WITH_NO_FOLLOWING_SPACE!
					(IDENTIFIER	(options{greedy=true;}:ASSIGN_WITH_NO_LEADING_SPACE)?
					|FUNCTION	(options{greedy=true;}:ASSIGN_WITH_NO_LEADING_SPACE)?
					|CONSTANT	(options{greedy=true;}:ASSIGN_WITH_NO_LEADING_SPACE)?
					|GLOBAL_VARIABLE 
					|INSTANCE_VARIABLE
					|CLASS_VARIABLE
					|UNARY_PLUS_MINUS_METHOD_NAME
					|operatorAsMethodname
					|keyword
					|string)
			{tellLexerWeHaveFinishedParsingSymbol();}
			{#symbol = #(#[SYMBOL, "SYMBOL"], #symbol);}
		;

/*
 *Operator expressions
 *lowest->
 *		and or
 *		not
 *		? :
 *		= += -= *= /= %= &= ^= |= <<= >>= **=
 *		.. ...
 *		||
 *		&&
 *		<=> ==  === !=  =~  !~
 *		>  >=  <  <=
 *		|	 ^
 *		&
 *		<<  >>
 *		+  -
 *		*  /  %
 *		-(unary)  +(unary)  !  ~
 *		**
 *		[]
 *		::
 *highest<-
 */

expression
		:	andorExpression
		;

//and or
andorExpression
		:	notExpression
			(options{greedy=true;/*caused by command*/}:
				(	"and"^		(options{greedy=true;}:LINE_BREAK!)?
				|	"or"^		(options{greedy=true;}:LINE_BREAK!)?
				)
				notExpression
			)*
		;

//not
notExpression
		:	"not"^
			(options{greedy=true;}:LINE_BREAK!)?
			notExpression
		|	ternaryIfThenElseExpression
		;

ternaryIfThenElseExpression
		:	assignmentExpression
			(options{greedy=true;/*caused by command*/}:
				QUESTION^			(options{greedy=true;}:LINE_BREAK!)?
				ternaryIfThenElseExpression
				(COLON!|COLON_WITH_NO_FOLLOWING_SPACE!)				(options{greedy=true;}:LINE_BREAK!)?
				ternaryIfThenElseExpression
			)?
		;

//= += -= *= /= %= **= &= ^= |= <<= >>= &&= ||=
assignmentExpression
		:	rangeExpression
			(options{greedy=true;/*caused by command*/}:
			{REST_ARG_PREFIX != LA(2)}?
				(	ASSIGN^				(options{greedy=true;}:LINE_BREAK!)?
				|	a:ASSIGN_WITH_NO_LEADING_SPACE^	{{#a.setType(ASSIGN);}}	(options{greedy=true;}:LINE_BREAK!)?
				|	PLUS_ASSIGN^		(options{greedy=true;}:LINE_BREAK!)?
				|	MINUS_ASSIGN^		(options{greedy=true;}:LINE_BREAK!)?
				|	STAR_ASSIGN^		(options{greedy=true;}:LINE_BREAK!)?
				|	DIV_ASSIGN^		(options{greedy=true;}:LINE_BREAK!)?
				|	MOD_ASSIGN^		(options{greedy=true;}:LINE_BREAK!)?
				|	POWER_ASSIGN^		(options{greedy=true;}:LINE_BREAK!)?
				|	BAND_ASSIGN^		(options{greedy=true;}:LINE_BREAK!)?
				|	BXOR_ASSIGN^		(options{greedy=true;}:LINE_BREAK!)?
				|	BOR_ASSIGN^		(options{greedy=true;}:LINE_BREAK!)?
				|	LEFT_SHIFT_ASSIGN^	(options{greedy=true;}:LINE_BREAK!)?
				|	RIGHT_SHIFT_ASSIGN^	(options{greedy=true;}:LINE_BREAK!)?
				|	LOGICAL_AND_ASSIGN^	(options{greedy=true;}:LINE_BREAK!)?
				|	LOGICAL_OR_ASSIGN^	(options{greedy=true;}:LINE_BREAK!)?
				)
				assignmentExpression
			)*
		;


//.. ...
rangeExpression
		:	logicalOrExpression
			(options{greedy=true;/*caused by command*/}:
				(	INCLUSIVE_RANGE^	(options{greedy=true;}:LINE_BREAK!)?
				|	EXCLUSIVE_RANGE^	(options{greedy=true;}:LINE_BREAK!)?
				)
				logicalOrExpression
			)* 
		;

//||
logicalOrExpression
		:	logicalAndExpression
			(options{greedy=true;/*caused by command*/}:
				LOGICAL_OR^		(options{greedy=true;}:LINE_BREAK!)?
				logicalAndExpression
			)*
		;

//&&
logicalAndExpression
		:	equalityExpression
			(options{greedy=true;/*caused by command*/}:
				LOGICAL_AND^		(options{greedy=true;}:LINE_BREAK!)?
				equalityExpression
			)*
		;

//<=> ==  === !=  =~  !~
equalityExpression
		:	relationalExpression
			(options{greedy=true;/*caused by command*/}:
				(	COMPARE^		(options{greedy=true;}:LINE_BREAK!)?
				|	EQUAL^			(options{greedy=true;}:LINE_BREAK!)?
				|	CASE_EQUAL^	(options{greedy=true;}:LINE_BREAK!)?
				|	NOT_EQUAL^		(options{greedy=true;}:LINE_BREAK!)?
				|	MATCH^			(options{greedy=true;}:LINE_BREAK!)?
				|	NOT_MATCH^		(options{greedy=true;}:LINE_BREAK!)?
				)
				relationalExpression
			)*
		;


//>  >=  <  <=
relationalExpression
		:	orExpression
			(options{greedy=true;/*caused by command*/}:
				(	LESS_THAN^			(options{greedy=true;}:LINE_BREAK!)?
				|	GREATER_THAN^		(options{greedy=true;}:LINE_BREAK!)?
				|	LESS_OR_EQUAL^		(options{greedy=true;}:LINE_BREAK!)?
				|	GREATER_OR_EQUAL^	(options{greedy=true;}:LINE_BREAK!)?
				)
				orExpression
			)*
		;

//|  ^
orExpression
		:	andExpression
			(options{greedy=true;/*caused by command*/}:
				(	BXOR^			(options{greedy=true;}:LINE_BREAK!)?
				|	BOR^			(options{greedy=true;}:LINE_BREAK!)?
				)
				andExpression
			)*
		;

//&
andExpression
		:	shiftExpression
			(options{greedy=true;/*caused by command*/}:
				BAND^			(options{greedy=true;}:LINE_BREAK!)?
				shiftExpression
			)*
		;



//<<  >>
shiftExpression
		:	additiveExpression
			(options{greedy=true;/*caused by command*/}:
				(	LEFT_SHIFT^		(options{greedy=true;}:LINE_BREAK!)?
				|	RIGHT_SHIFT^	(options{greedy=true;}:LINE_BREAK!)?
				)
				additiveExpression
			)*
		;



//+  -
additiveExpression
		:	multiplicativeExpression
			(options{greedy=true;/*caused by command*/}:
				(	PLUS^				(options{greedy=true;}:LINE_BREAK!)?
				|	MINUS^				(options{greedy=true;}:LINE_BREAK!)?
				)
				multiplicativeExpression
			)*
		;

//*  /  %
multiplicativeExpression
		:	powerExpression
			(options{greedy=true;/*caused by command*/}:
				(	STAR^			(options{greedy=true;}:LINE_BREAK!)?
				|	DIV^			(options{greedy=true;}:LINE_BREAK!)?
				|	MOD^			(options{greedy=true;}:LINE_BREAK!)?
				)
				powerExpression
			)*
		;


//**
powerExpression
		:	bnotExpression
			(options{greedy=true;/*caused by command*/}:
				POWER^			(options{greedy=true;}:LINE_BREAK!)?
				bnotExpression
			)*
		;

//!  ~
bnotExpression
		:	(	BNOT^			(options{greedy=true;}:LINE_BREAK!)?
			|	NOT^			(options{greedy=true;}:LINE_BREAK!)?
			)*
			command
		;

command
{
	boolean seen_arguments = false;
	boolean seen_block = false;
	boolean is_function = false;
}
		:	dotColonOrArrayAccess	{is_function = (2 == can_be_command_);}
			(
				{can_be_command_ > 0}?
				methodInvocationArgumentWithoutParen[false]	{seen_arguments = true;}
			)?
			(options{greedy=true;/*caused by command*/}:codeBlock	{seen_block = true;})?
			{
				if (seen_arguments || seen_block || (is_function && isNotAssign(LA(1))))
				{
					#command = #(#[CALL, "CALL"], #command);
				}
			}
		|	("return"^|"break"^|"next"^)	(options{greedy=true;/*caused by command*/}:methodInvocationArgumentWithoutParen[false])?
		;


dotColonOrArrayAccess
		:	unaryExpression
			(options{greedy=true;/*caused by command*/}:
				(DOT^		methodCall
				|COLON2^	methodCall
				|LBRACK_ARRAY_ACCESS^
					(arrayReferenceArgument)?
					RBRACK!
				|EMPTY_ARRAY_ACCESS
				)
				{can_be_command_ = 1;}
			)*
		;

//-(unary)  +(unary)
unaryExpression
		:	(	UNARY_PLUS^	(options{greedy=true;}:LINE_BREAK!)?
			|	UNARY_MINUS^	(options{greedy=true;}:LINE_BREAK!)?
			)*
			(methodCall
			|primaryExpressionThatCanNotBeMethodName	{can_be_command_ = 0;}
			)
		;

methodCall
{
	can_be_command_ = 1;
	boolean seen_arguments = false;
	boolean seen_block = false;
	boolean is_function = false;
}
		:	name:primaryExpressionThatCanBeMethodName	{is_function = (2 == can_be_command_);}
			(options{greedy=true;/*caused by command*/}:arg:methodInvocationArgumentWithParen	{seen_arguments = true;})?
			(options{greedy=true;/*caused by command*/}:codeBlock	{seen_block = true;})?
			{
				if (seen_arguments || seen_block || (is_function && (LA(1) == LBRACK_ARRAY_ACCESS)/*this is for a[1]*/))
				{
					#methodCall = #(#[CALL, "CALL"], #methodCall);
					can_be_command_ = 0;
				}
			}
		;

methodInvocationArgumentWithParen
		:	LPAREN!
				(methodInvocationArgumentWithoutParen[true])?
				(LINE_BREAK!)?
			RPAREN!
		;

methodInvocationArgumentWithoutParen[boolean should_ignore_line_break]
{
boolean seen_star_or_band = false;
}
		:	(normalMethodInvocationArgument[should_ignore_line_break]
			(options{greedy=true;/*caused by command*/}:
				COMMA!	{if (REST_ARG_PREFIX == LA(1) || BLOCK_ARG_PREFIX == LA(1))	{seen_star_or_band = true;break;}}
				normalMethodInvocationArgument[should_ignore_line_break]
			)*
			({seen_star_or_band}?	restMethodInvocationArgument
			|{seen_star_or_band}?	blockMethodInvocationArgument
			)?
			|	restMethodInvocationArgument
			|	blockMethodInvocationArgument
			)
			{#methodInvocationArgumentWithoutParen = #(#[ARG, "ARG"], #methodInvocationArgumentWithoutParen);}
		;

normalMethodInvocationArgument[boolean should_ignore_line_break]
		:	expression	(options{greedy=true;/*caused by command*/}:ASSOC^	expression)?	({should_ignore_line_break}?	LINE_BREAK!)?
		;

restMethodInvocationArgument
		:	REST_ARG_PREFIX	expression	(options{greedy=true;/*caused by command*/}:COMMA!	blockMethodInvocationArgument)?
		;

blockMethodInvocationArgument
		:	BLOCK_ARG_PREFIX	expression
		;

predefinedValue
		:	"nil"
		|	"true"
		|	"false"
		|	"__FILE__"
		|	"__LINE__"
		;

string
		:	DOUBLE_QUOTE_STRING
		|	SINGLE_QUOTE_STRING
		|	STRING_BEFORE_EXPRESSION_SUBSTITUTION^
			expression_substituation	{tellLexerWeHaveFinishedParsingStringExpressionSubstituation();}
			(STRING_BETWEEN_EXPRESSION_SUBSTITUTION	expression_substituation	{tellLexerWeHaveFinishedParsingStringExpressionSubstituation();})*
			STRING_AFTER_EXPRESSION_SUBSTITUTION
		;

protected
expression_substituation
		:	(	LCURLY_BLOCK!	(compoundStatement)?	RCURLY!
			|	GLOBAL_VARIABLE 
			|	INSTANCE_VARIABLE
			|	CLASS_VARIABLE
			)
		;

regex
		:	REGEX
		|	REGEX_BEFORE_EXPRESSION_SUBSTITUTION^
			expression_substituation	{tellLexerWeHaveFinishedParsingRegexExpressionSubstituation();}
			(STRING_BETWEEN_EXPRESSION_SUBSTITUTION	expression_substituation	{tellLexerWeHaveFinishedParsingRegexExpressionSubstituation();})*
			STRING_AFTER_EXPRESSION_SUBSTITUTION
		;

command_output
		:	COMMAND_OUTPUT
		|	COMMAND_OUTPUT_BEFORE_EXPRESSION_SUBSTITUTION^
			expression_substituation	{tellLexerWeHaveFinishedParsingStringExpressionSubstituation();}
			(STRING_BETWEEN_EXPRESSION_SUBSTITUTION	expression_substituation	{tellLexerWeHaveFinishedParsingStringExpressionSubstituation();})*
			STRING_AFTER_EXPRESSION_SUBSTITUTION
		;

heredoc
		:	HERE_DOC_BEGIN!
			(HERE_DOC_CONTENT
			|HERE_DOC_BEFORE_EXPRESSION_SUBSTITUTION^
			expression_substituation	{tellLexerWeHaveFinishedParsingHeredocExpressionSubstituation();}
			(HERE_DOC_BETWEEN_EXPRESSION_SUBSTITUTION	expression_substituation	{tellLexerWeHaveFinishedParsingHeredocExpressionSubstituation();})*
			HERE_DOC_AFTER_EXPRESSION_SUBSTITUTION
			)
		;

literal
		:	regex
		|	(options{greedy=true;/*caused by command*/}:string)+{ #literal = #([STRING, "STRING"], #literal); }
		|	heredoc
		|	command_output
		|	symbol
		|	W_ARRAY
		;

numeric
		:	INTEGER
		|	HEX		
		|	BINARY
		|	OCTAL
		|	FLOAT
		|	ASCII_VALUE
		;

primaryExpressionThatCanBeMethodName
		:	FUNCTION	{can_be_command_ = 2;}
		|	IDENTIFIER		//this is OK: p = 1; p 1
		|	"self"
		|	"super"		{can_be_command_ = 2;}
		|	CONSTANT
		|	LEADING_COLON2	CONSTANT
		|	"retry"
		|	"yield"		{can_be_command_ = 2;}
		|	keyword_defined
		|	"redo"
		|	EMPTY_ARRAY
		|	EMPTY_ARRAY_ACCESS
		|	UNARY_PLUS_MINUS_METHOD_NAME
		;

primaryExpressionThatCanNotBeMethodName
		:	INSTANCE_VARIABLE	
		|	GLOBAL_VARIABLE 
		|	CLASS_VARIABLE
		|	predefinedValue
		|	numeric
		|	literal
		|	arrayExpression
		|	hashExpression
		|	LPAREN^ compoundStatement RPAREN!
		|	ifExpression
		|	unlessExpression
		|	whileExpression			
		|	untilExpression			
		|	caseExpression		
		|	forExpression			
		|	exceptionHandlingExpression
		|	moduleDefination
		|	classDefination
		|	methodDefination
		;

primaryExpression
		:	primaryExpressionThatCanNotBeMethodName
		|	primaryExpressionThatCanBeMethodName	
		;

arrayReferenceArgument
		:	keyValuePair
			(	COMMA! {if (ASSIGN == LA(1)||RBRACK == LA(1)) break;}
				(seen_star:REST_ARG_PREFIX)?	keyValuePair	{if (null != seen_star) break;}
			)*
		|	REST_ARG_PREFIX	expression	(LINE_BREAK!)?
		;

arrayExpression
		:	LBRACK^
				(arrayReferenceArgument)?
			RBRACK!
		;

keyValuePair
		:	expression	(ASSOC	expression)?	(LINE_BREAK!)?
		;

hashExpression
		:	LCURLY_HASH^
				(
					keyValuePair
					(options{greedy=true;}:{RCURLY != LA(2)}?	COMMA!	keyValuePair)*
					(COMMA!)?
				)?
			RCURLY!
		;

bodyStatement
			:	(compoundStatement
				("rescue" exceptionList thenOrTermialOrColon (compoundStatement)?)*
				("else" (compoundStatement)?)?
				("ensure" (compoundStatement)?)?
			|	("rescue" exceptionList thenOrTermialOrColon (compoundStatement)?)+
				("else" (compoundStatement)?)?
				("ensure" (compoundStatement)?)?
			|	("ensure" (compoundStatement)?)
			)
			{#bodyStatement = #(#[BODY, "BODY"], #bodyStatement);}
		;

exceptionHandlingExpression
		:	"begin"^
			(bodyStatement)?
			"end"!
		;

exceptionList
		:	((className|INSTANCE_VARIABLE|IDENTIFIER)	(COMMA!	(className|INSTANCE_VARIABLE|IDENTIFIER))*)?	(ASSOC	(IDENTIFIER|func:FUNCTION{addVariable(func);}))?
		;

ifExpression
		:	"if"^		(LINE_BREAK!)?
			expression thenOrTermialOrColon (compoundStatement)?
			("elsif" (expression)? thenOrTermialOrColon (compoundStatement)?)*
			("else" (compoundStatement)?)?
			"end"!
		;

unlessExpression
		:	"unless"^		(LINE_BREAK!)?
			expression thenOrTermialOrColon (compoundStatement)?
			("else" (compoundStatement)?)?
			"end"!
		;

caseExpression
		:	"case"^ (expression)? thenOrTermialOrColon		//(compoundStatement)?
			(keyword_when mrhs	thenOrTermialOrColon (compoundStatement)?)+
			("else" (compoundStatement)?)?
			"end"!
		;

forExpression
		:	"for"^	(LINE_BREAK!)?
			block_vars
			"in"		(LINE_BREAK!)?
			expression doOrTermialOrColon
			(compoundStatement)?
			"end"!
		;

whileExpression
		:	"while"^	(LINE_BREAK!)?
			expression doOrTermialOrColon
			(compoundStatement)?
			"end"!
		;

untilExpression
		:	"until"^	(LINE_BREAK!)?
			expression doOrTermialOrColon
			(compoundStatement)?
			"end"!
		;

moduleDefination
		:	"module"^	(LINE_BREAK!)?
			moduleName (options {greedy=true;}:terminal)?	{enterScope();}
			(bodyStatement)?
			"end"!			{leaveScope();}
		;

moduleName
		:	CONSTANT	(COLON2 CONSTANT)*
		;

classDefination
		:	"class"^	(LINE_BREAK!)?
			(	className	(LESS_THAN expression)?
			|	LEFT_SHIFT	expression
			)
			terminal			{enterScope();}
			(bodyStatement)?
			"end"!			{leaveScope();}
		;

/*
FIXME
cname	: tIDENTIFIER 
		| tCONSTANT
		;

cpath	: tCOLON3 cname 
		| cname 
		| primary_value tCOLON2 cname 
		;
*/
className
		:	(CONSTANT|FUNCTION)	(COLON2^	CONSTANT)*
		|	(LEADING_COLON2	CONSTANT)	(COLON2^	CONSTANT)*
		;

methodDefination
		:	"def"^		(options{greedy=true;}:LINE_BREAK!)?
			methodName	{enterScope();}
			methodDefinationArgument
			(bodyStatement)?
			"end"!		{leaveScope();}
		;

protected
methodNameSupplement
		:	(DOT|COLON2)
			(IDENTIFIER	(ASSIGN_WITH_NO_LEADING_SPACE)?
			|FUNCTION	(ASSIGN_WITH_NO_LEADING_SPACE)?
			|CONSTANT	(ASSIGN_WITH_NO_LEADING_SPACE)?
			|operatorAsMethodname
			|keyword
			)
		;

methodName
{boolean is_singleton_method = false;}
		:	(	operatorAsMethodname
			|	keywordAsMethodName
			|	(id:IDENTIFIER
				|function:FUNCTION	
				|constant:CONSTANT	
				)
				(methodNameSupplement{is_singleton_method = true;}|ASSIGN_WITH_NO_LEADING_SPACE)?
			|	LPAREN
				expression	(LINE_BREAK!)?
				RPAREN
				methodNameSupplement{is_singleton_method = true;}
			|	(INSTANCE_VARIABLE|CLASS_VARIABLE|GLOBAL_VARIABLE )	methodNameSupplement	{is_singleton_method = true;}
			|	("nil"|"self"|"true"|"false"|"__FILE__"|"__LINE__")	methodNameSupplement	{is_singleton_method = true;}
			|	UNARY_PLUS_MINUS_METHOD_NAME
			)
			{
				if (is_singleton_method) {
					#methodName = #(#[SINGLETON_METHOD, "SINGLETON_METHOD"], #methodName);
				}
			}
		;

operatorAsMethodname
		:	LEFT_SHIFT
		|	RIGHT_SHIFT
		|	EQUAL
		|	CASE_EQUAL
		|	GREATER_THAN
		|	GREATER_OR_EQUAL
		|	LESS_THAN
		|	LESS_OR_EQUAL
		|	PLUS
		|	MINUS
		|	STAR
		|	DIV
		|	MOD
		|	POWER
		|	BAND
		|	BOR
		|	BXOR
		|	(EMPTY_ARRAY	|EMPTY_ARRAY_ACCESS)	(options{greedy=true;}:ASSIGN_WITH_NO_LEADING_SPACE)?
		|	MATCH
		|	COMPARE
		|	BNOT
		|	SINGLE_QUOTE
		;

keyword
		:	keywordAsMethodName
		|	"nil"
		|	"self"
		|	"true"
		|	"false"
		|	"__FILE__"
		|	"__LINE__"
		;

keywordAsMethodName
		:	"and"
		|	"def"
		|	"end"
		|	"in"
		|	"or"
		|	"unless"
		|	"begin"
		|	"defined?"
		|	"ensure"
		|	"module"
		|	"redo"
		|	"super"
		|	"until"
		|	"BEGIN"
		|	"break"
		|	"do"
		|	"next"
		|	"rescue"
		|	"then"
		|	"when"
		|	"END"
		|	"case"
		|	"else"
		|	"for"
		|	"retry"
		|	"while"
		|	"alias"
		|	"class"
		|	"elsif"
		|	"if"
		|	"not"
		|	"return"
		|	"undef"
		|	"yield"
		;

methodDefinationArgument
		:	LPAREN!
				(methodDefinationArgumentWithoutParen)?
			RPAREN!	{tellLexerWeHaveFinishedParsingMethodparameters();}
			(options {greedy=true;}:terminal)?
		|	(methodDefinationArgumentWithoutParen)?	terminal
		;

methodDefinationArgumentWithoutParen
{boolean seen_star_or_band = false;}
		:	normalMethodDefinationArgument
				(COMMA!	{if (REST_ARG_PREFIX == LA(1) || BLOCK_ARG_PREFIX == LA(1))	{seen_star_or_band = true;break;}}	normalMethodDefinationArgument)*
				(	{seen_star_or_band}?	restMethodDefinationArgument
					|{seen_star_or_band}?	blockMethodDefinationArgument
				)?
		|	restMethodDefinationArgument
		|	blockMethodDefinationArgument
		;

normalMethodDefinationArgument
		:	(id1:IDENTIFIER{addVariable(id1);}|id2:FUNCTION{addVariable(id2);})	((ASSIGN|ASSIGN_WITH_NO_LEADING_SPACE)	expression)?
		;

restMethodDefinationArgument
		:	REST_ARG_PREFIX	((id1:IDENTIFIER{addVariable(id1);}|id2:FUNCTION{addVariable(id2);})	(COMMA!	blockMethodDefinationArgument)?)?
		;

blockMethodDefinationArgument
		:	BLOCK_ARG_PREFIX	(id1:IDENTIFIER{addVariable(id1);}|id2:FUNCTION{addVariable(id2);})
		;


thenOrTermialOrColon
		:	"then"!
		|	terminal	("then"!)?
		|	COLON!
		;

doOrTermialOrColon
		:	DO_IN_CONDITION!
		|	terminal
		|	COLON!
		;

operator_ASSIGN				:	(ASSIGN|ASSIGN_WITH_NO_LEADING_SPACE)				(options{greedy=true;}:LINE_BREAK!)?;
/*
operator_QUESTION			:	QUESTION			(options{greedy=true;}:LINE_BREAK!)?;
operator_COLON				:	(COLON|COLON_WITH_NO_FOLLOWING_SPACE)			(options{greedy=true;}:LINE_BREAK!)?;
operator_PLUS_ASSIGN			:	PLUS_ASSIGN			(options{greedy=true;}:LINE_BREAK!)?;
operator_MINUS_ASSIGN		:	MINUS_ASSIGN		(options{greedy=true;}:LINE_BREAK!)?;
operator_STAR_ASSIGN		:	STAR_ASSIGN		(options{greedy=true;}:LINE_BREAK!)?;
operator_DIV_ASSIGN			:	DIV_ASSIGN			(options{greedy=true;}:LINE_BREAK!)?;
operator_MOD_ASSIGN			:	MOD_ASSIGN			(options{greedy=true;}:LINE_BREAK!)?;
operator_POWER_ASSIGN		:	POWER_ASSIGN		(options{greedy=true;}:LINE_BREAK!)?;
operator_BAND_ASSIGN		:	BAND_ASSIGN		(options{greedy=true;}:LINE_BREAK!)?;
operator_BXOR_ASSIGN		:	BXOR_ASSIGN		(options{greedy=true;}:LINE_BREAK!)?;
operator_BOR_ASSIGN			:	BOR_ASSIGN			(options{greedy=true;}:LINE_BREAK!)?;
operator_LEFT_SHIFT_ASSIGN	:	LEFT_SHIFT_ASSIGN	(options{greedy=true;}:LINE_BREAK!)?;
operator_RIGHT_SHIFT_ASSIGN	:	RIGHT_SHIFT_ASSIGN	(options{greedy=true;}:LINE_BREAK!)?;<commit_msg>minor clean up, create 'variable' rule<commit_after>header {
package com.xruby.compiler.parser;
}

/** 
 * Copyright (c) 2005-2006 <NAME>. All rights reserved.
 */

//----------------------------------------------------------------------------
// The Ruby parser. Do not use this one, use RubyParser instead!
//----------------------------------------------------------------------------
class RubyParserBase extends Parser;
options {
	k = 1;                           // Will add prediction manually if we need k > 1.
	exportVocab=Ruby;
	codeGenMakeSwitchThreshold = 2;
	codeGenBitsetTestThreshold = 999;
	defaultErrorHandler = false;
	buildAST = true;
	//analyzerDebug = true;
}

tokens {
	// "imaginary" tokens
	RPAREN_IN_METHOD_DEFINATION;
	BODY;
	CALL;
	ARG;
	COMPSTMT;
	SYMBOL;
	BLOCK;
	MULTIPLE_ASSIGN;
	MULTIPLE_ASSIGN_WITH_EXTRA_COMMA;
	MRHS;
	NESTED_LHS;
	SINGLETON_METHOD;
	STRING;
}

{
	private int can_be_command_ = 0;

	protected void enterScope()	{assert(false);}
	protected void enterBlockScope()	{assert(false);}
	protected void leaveScope()	{assert(false);}
	protected void addVariable(Token id)	{assert(false);}
	protected void tellLexerWeHaveFinishedParsingMethodparameters()	{assert(false);}
	protected void tellLexerWeHaveFinishedParsingSymbol()	{assert(false);}
	protected void tellLexerWeHaveFinishedParsingStringExpressionSubstituation()	{assert(false);}
	protected void tellLexerWeHaveFinishedParsingRegexExpressionSubstituation()	{assert(false);}
	protected void tellLexerWeHaveFinishedParsingHeredocExpressionSubstituation()	{assert(false);}

	boolean isNotAssign(int token_type) {
		switch (token_type) {
			case ASSIGN:
			case ASSIGN_WITH_NO_LEADING_SPACE:
			case PLUS_ASSIGN:
			case MINUS_ASSIGN:
			case STAR_ASSIGN:
			case DIV_ASSIGN:
			case MOD_ASSIGN:
			case POWER_ASSIGN:
			case BAND_ASSIGN:
			case BXOR_ASSIGN:
			case BOR_ASSIGN:
			case LEFT_SHIFT_ASSIGN:
			case RIGHT_SHIFT_ASSIGN:
			case LOGICAL_AND_ASSIGN:
			case LOGICAL_OR_ASSIGN:
				return false;
			default:
				return true;
		}
	}
}

program
		:	(compoundStatement)? EOF!
		;

compoundStatement
{boolean empty = true;}
		:	(terminal!	(statements	{empty = false;})?
			|	statements	{empty = false;})
			{if (!empty)	#compoundStatement = #(#[COMPSTMT, "COMPSTMT"], #compoundStatement);}
		;

statements
		:	statement
			(
				terminal
				{
					if (	(EOF == LA(1))	||	//script end
						(RCURLY == LA(1))	||	(LITERAL_end == LA(1))	||	//block end
						(RPAREN == LA(1))	||
						(LITERAL_else == LA(1)) || (LITERAL_elsif == LA(1))	||
						(LITERAL_rescue == LA(1)) || (LITERAL_ensure == LA(1))	||
						(LITERAL_when == LA(1))
					)
					break;
				}
				statement
			)*
		;

terminal
		:	SEMI!
		|	LINE_BREAK!
		;

statement
		:	body:statementWithoutModifier
									(!IF_MODIFIER^		if_condition:expression	{#statement = #(#[LITERAL_if, "if"], if_condition, #(#[COMPSTMT, "COMPSTMT"], #body));}
									|!UNLESS_MODIFIER^	unless_condition:expression	{#statement = #(#[LITERAL_unless, "unless"], unless_condition, #(#[COMPSTMT, "COMPSTMT"], #body));}
									|!WHILE_MODIFIER^	while_condition:expression	{#statement = #(#[LITERAL_while, "while"], while_condition, #(#[COMPSTMT, "COMPSTMT"], #body));}
									|!UNTIL_MODIFIER^	until_condition:expression	{#statement = #(#[LITERAL_until, "until"], until_condition, #(#[COMPSTMT, "COMPSTMT"], #body));}
									|!RESCUE_MODIFIER^	rescue_condition:expression	{#statement = #(#[LITERAL_rescue, "rescue"], rescue_condition, #(#[COMPSTMT, "COMPSTMT"], #body));}
									)*
		;

statementWithoutModifier
{
	boolean is_parallel_assignment = false;
	boolean has_extra_comma = false;
}
		:	(aliasStatement
			|	undefStatement
			|	keyword_BEGIN 	LCURLY_BLOCK (compoundStatement)? RCURLY
			|	keyword_END 	LCURLY_BLOCK (compoundStatement)? RCURLY
			|	expression
				(options{greedy=true;}:
					COMMA!
					{if (ASSIGN == LA(1) || ASSIGN_WITH_NO_LEADING_SPACE == LA(1)) {has_extra_comma=true;break;}}
					(seen_star:REST_ARG_PREFIX)?
					(mlhs_item)?	{if (null != seen_star) break;}
					{is_parallel_assignment = true;}
				)*
				(	(ASSIGN!|ASSIGN_WITH_NO_LEADING_SPACE!)
					(options{greedy=true;}:LINE_BREAK!)?
					mrhs
					{is_parallel_assignment = true;}
				)?
			|	REST_ARG_PREFIX
				mlhs_item
				(ASSIGN!|ASSIGN_WITH_NO_LEADING_SPACE!)
				(options{greedy=true;}:LINE_BREAK!)?
				mrhs
				{is_parallel_assignment = true;}
			)
			{	
				if (is_parallel_assignment && !has_extra_comma) {
					#statementWithoutModifier = #(#[MULTIPLE_ASSIGN, "MULTIPLE_ASSIGN"], #statementWithoutModifier);
				} else if (is_parallel_assignment && has_extra_comma) {
					#statementWithoutModifier = #(#[MULTIPLE_ASSIGN_WITH_EXTRA_COMMA, "MULTIPLE_ASSIGN_WITH_EXTRA_COMMA"], #statementWithoutModifier);
				}
			}
		;

//LPAREN can start primaryExpression, have to use syntactic predicate here.
mlhs_item
		:	(LPAREN!	dotColonOrArrayAccess	COMMA)=>
			LPAREN!
			dotColonOrArrayAccess
			(COMMA!	(seen_star:REST_ARG_PREFIX)?	dotColonOrArrayAccess	{if (null != seen_star) break;})+
			RPAREN!
			{#mlhs_item = #(#[NESTED_LHS, "NESTED_LHS"], #mlhs_item);}
		|	dotColonOrArrayAccess
		;

mrhs
		:	(expression	(COMMA! {if (ASSIGN == LA(1)||RBRACK == LA(1)) break;}	(seen_star:REST_ARG_PREFIX)?	expression	{if (null != seen_star) break;})*
			|	REST_ARG_PREFIX	expression
			)
			{#mrhs = #(#[MRHS, "MRHS"], #mrhs);}
		;

protected
undef_parameter
		:	IDENTIFIER	(ASSIGN_WITH_NO_LEADING_SPACE)?
		|	CONSTANT	(ASSIGN_WITH_NO_LEADING_SPACE)?
		|	FUNCTION	(ASSIGN_WITH_NO_LEADING_SPACE)?
		|	symbol
		|	keywordAsMethodName
		|	operatorAsMethodname
		;

undefStatement
		:	"undef"^	(options{greedy=true;}:LINE_BREAK!)?
			undef_parameter	(COMMA!	undef_parameter)*
		;

aliasStatement
		:	"alias"^	(options{greedy=true;}:LINE_BREAK!)?
			(GLOBAL_VARIABLE 	(LINE_BREAK!)?	GLOBAL_VARIABLE 
			|undef_parameter	(LINE_BREAK!)?	undef_parameter
			)
		;

//LPAREN can start primaryExpression, have to use syntactic predicate here.
block_var
		:	(LPAREN!	dotColonOrArrayAccess	COMMA!)=>	LPAREN	dotColonOrArrayAccess	(COMMA!	{if (RPAREN == LA(1)) break;}	(seen_star:REST_ARG_PREFIX)?	dotColonOrArrayAccess	{if (null != seen_star) break;})*		RPAREN!
		|	REST_ARG_PREFIX	(dotColonOrArrayAccess)?
		|	dotColonOrArrayAccess
		;

//TODO should be mlhs
block_vars
		:	block_var		(COMMA!	{if (BOR == LA(1) || LITERAL_in == LA(1)) break;}	block_var	)*
		;

codeBlock
		:	("do"!	(options{greedy=true;}:LINE_BREAK!)?		{enterBlockScope();}
				blockContent
				"end"!			{leaveScope();}
			|	LCURLY_BLOCK!	{enterBlockScope();}
				blockContent
				RCURLY!			{leaveScope();}
			)
			{#codeBlock = #(#[BLOCK, "BLOCK"], #codeBlock);}
		;

protected
blockContent
		:	(BOR	(block_vars)?	BOR!
			|LOGICAL_OR
			)?	(compoundStatement)?
		;

symbol
		:	COLON_WITH_NO_FOLLOWING_SPACE!
					(IDENTIFIER	(options{greedy=true;}:ASSIGN_WITH_NO_LEADING_SPACE)?
					|FUNCTION	(options{greedy=true;}:ASSIGN_WITH_NO_LEADING_SPACE)?
					|CONSTANT	(options{greedy=true;}:ASSIGN_WITH_NO_LEADING_SPACE)?
					|GLOBAL_VARIABLE 
					|INSTANCE_VARIABLE
					|CLASS_VARIABLE
					|UNARY_PLUS_MINUS_METHOD_NAME
					|operatorAsMethodname
					|keyword
					|string)
			{tellLexerWeHaveFinishedParsingSymbol();}
			{#symbol = #(#[SYMBOL, "SYMBOL"], #symbol);}
		;

/*
 *Operator expressions
 *lowest->
 *		and or
 *		not
 *		? :
 *		= += -= *= /= %= &= ^= |= <<= >>= **=
 *		.. ...
 *		||
 *		&&
 *		<=> ==  === !=  =~  !~
 *		>  >=  <  <=
 *		|	 ^
 *		&
 *		<<  >>
 *		+  -
 *		*  /  %
 *		-(unary)  +(unary)  !  ~
 *		**
 *		[]
 *		::
 *highest<-
 */

expression
		:	andorExpression
		;

//and or
andorExpression
		:	notExpression
			(options{greedy=true;/*caused by command*/}:
				(	"and"^		(options{greedy=true;}:LINE_BREAK!)?
				|	"or"^		(options{greedy=true;}:LINE_BREAK!)?
				)
				notExpression
			)*
		;

//not
notExpression
		:	"not"^
			(options{greedy=true;}:LINE_BREAK!)?
			notExpression
		|	ternaryIfThenElseExpression
		;

ternaryIfThenElseExpression
		:	assignmentExpression
			(options{greedy=true;/*caused by command*/}:
				QUESTION^			(options{greedy=true;}:LINE_BREAK!)?
				ternaryIfThenElseExpression
				(COLON!|COLON_WITH_NO_FOLLOWING_SPACE!)				(options{greedy=true;}:LINE_BREAK!)?
				ternaryIfThenElseExpression
			)?
		;

//= += -= *= /= %= **= &= ^= |= <<= >>= &&= ||=
assignmentExpression
		:	rangeExpression
			(options{greedy=true;/*caused by command*/}:
			{REST_ARG_PREFIX != LA(2)}?
				(	ASSIGN^				(options{greedy=true;}:LINE_BREAK!)?
				|	a:ASSIGN_WITH_NO_LEADING_SPACE^	{{#a.setType(ASSIGN);}}	(options{greedy=true;}:LINE_BREAK!)?
				|	PLUS_ASSIGN^		(options{greedy=true;}:LINE_BREAK!)?
				|	MINUS_ASSIGN^		(options{greedy=true;}:LINE_BREAK!)?
				|	STAR_ASSIGN^		(options{greedy=true;}:LINE_BREAK!)?
				|	DIV_ASSIGN^		(options{greedy=true;}:LINE_BREAK!)?
				|	MOD_ASSIGN^		(options{greedy=true;}:LINE_BREAK!)?
				|	POWER_ASSIGN^		(options{greedy=true;}:LINE_BREAK!)?
				|	BAND_ASSIGN^		(options{greedy=true;}:LINE_BREAK!)?
				|	BXOR_ASSIGN^		(options{greedy=true;}:LINE_BREAK!)?
				|	BOR_ASSIGN^		(options{greedy=true;}:LINE_BREAK!)?
				|	LEFT_SHIFT_ASSIGN^	(options{greedy=true;}:LINE_BREAK!)?
				|	RIGHT_SHIFT_ASSIGN^	(options{greedy=true;}:LINE_BREAK!)?
				|	LOGICAL_AND_ASSIGN^	(options{greedy=true;}:LINE_BREAK!)?
				|	LOGICAL_OR_ASSIGN^	(options{greedy=true;}:LINE_BREAK!)?
				)
				assignmentExpression
			)*
		;


//.. ...
rangeExpression
		:	logicalOrExpression
			(options{greedy=true;/*caused by command*/}:
				(	INCLUSIVE_RANGE^	(options{greedy=true;}:LINE_BREAK!)?
				|	EXCLUSIVE_RANGE^	(options{greedy=true;}:LINE_BREAK!)?
				)
				logicalOrExpression
			)* 
		;

//||
logicalOrExpression
		:	logicalAndExpression
			(options{greedy=true;/*caused by command*/}:
				LOGICAL_OR^		(options{greedy=true;}:LINE_BREAK!)?
				logicalAndExpression
			)*
		;

//&&
logicalAndExpression
		:	equalityExpression
			(options{greedy=true;/*caused by command*/}:
				LOGICAL_AND^		(options{greedy=true;}:LINE_BREAK!)?
				equalityExpression
			)*
		;

//<=> ==  === !=  =~  !~
equalityExpression
		:	relationalExpression
			(options{greedy=true;/*caused by command*/}:
				(	COMPARE^		(options{greedy=true;}:LINE_BREAK!)?
				|	EQUAL^			(options{greedy=true;}:LINE_BREAK!)?
				|	CASE_EQUAL^	(options{greedy=true;}:LINE_BREAK!)?
				|	NOT_EQUAL^		(options{greedy=true;}:LINE_BREAK!)?
				|	MATCH^			(options{greedy=true;}:LINE_BREAK!)?
				|	NOT_MATCH^		(options{greedy=true;}:LINE_BREAK!)?
				)
				relationalExpression
			)*
		;


//>  >=  <  <=
relationalExpression
		:	orExpression
			(options{greedy=true;/*caused by command*/}:
				(	LESS_THAN^			(options{greedy=true;}:LINE_BREAK!)?
				|	GREATER_THAN^		(options{greedy=true;}:LINE_BREAK!)?
				|	LESS_OR_EQUAL^		(options{greedy=true;}:LINE_BREAK!)?
				|	GREATER_OR_EQUAL^	(options{greedy=true;}:LINE_BREAK!)?
				)
				orExpression
			)*
		;

//|  ^
orExpression
		:	andExpression
			(options{greedy=true;/*caused by command*/}:
				(	BXOR^			(options{greedy=true;}:LINE_BREAK!)?
				|	BOR^			(options{greedy=true;}:LINE_BREAK!)?
				)
				andExpression
			)*
		;

//&
andExpression
		:	shiftExpression
			(options{greedy=true;/*caused by command*/}:
				BAND^			(options{greedy=true;}:LINE_BREAK!)?
				shiftExpression
			)*
		;



//<<  >>
shiftExpression
		:	additiveExpression
			(options{greedy=true;/*caused by command*/}:
				(	LEFT_SHIFT^		(options{greedy=true;}:LINE_BREAK!)?
				|	RIGHT_SHIFT^	(options{greedy=true;}:LINE_BREAK!)?
				)
				additiveExpression
			)*
		;



//+  -
additiveExpression
		:	multiplicativeExpression
			(options{greedy=true;/*caused by command*/}:
				(	PLUS^				(options{greedy=true;}:LINE_BREAK!)?
				|	MINUS^				(options{greedy=true;}:LINE_BREAK!)?
				)
				multiplicativeExpression
			)*
		;

//*  /  %
multiplicativeExpression
		:	powerExpression
			(options{greedy=true;/*caused by command*/}:
				(	STAR^			(options{greedy=true;}:LINE_BREAK!)?
				|	DIV^			(options{greedy=true;}:LINE_BREAK!)?
				|	MOD^			(options{greedy=true;}:LINE_BREAK!)?
				)
				powerExpression
			)*
		;


//**
powerExpression
		:	bnotExpression
			(options{greedy=true;/*caused by command*/}:
				POWER^			(options{greedy=true;}:LINE_BREAK!)?
				bnotExpression
			)*
		;

//!  ~
bnotExpression
		:	(	BNOT^			(options{greedy=true;}:LINE_BREAK!)?
			|	NOT^			(options{greedy=true;}:LINE_BREAK!)?
			)*
			command
		;

command
{
	boolean seen_arguments = false;
	boolean seen_block = false;
	boolean is_function = false;
}
		:	dotColonOrArrayAccess	{is_function = (2 == can_be_command_);}
			(
				{can_be_command_ > 0}?
				methodInvocationArgumentWithoutParen[false]	{seen_arguments = true;}
			)?
			(options{greedy=true;/*caused by command*/}:codeBlock	{seen_block = true;})?
			{
				if (seen_arguments || seen_block || (is_function && isNotAssign(LA(1))))
				{
					#command = #(#[CALL, "CALL"], #command);
				}
			}
		|	("return"^|"break"^|"next"^)	(options{greedy=true;/*caused by command*/}:methodInvocationArgumentWithoutParen[false])?
		;


dotColonOrArrayAccess
		:	unaryExpression
			(options{greedy=true;/*caused by command*/}:
				(DOT^		methodCall
				|COLON2^	methodCall
				|LBRACK_ARRAY_ACCESS^
					(arrayReferenceArgument)?
					RBRACK!
				|EMPTY_ARRAY_ACCESS
				)
				{can_be_command_ = 1;}
			)*
		;

//-(unary)  +(unary)
unaryExpression
		:	(	UNARY_PLUS^	(options{greedy=true;}:LINE_BREAK!)?
			|	UNARY_MINUS^	(options{greedy=true;}:LINE_BREAK!)?
			)*
			(methodCall
			|primaryExpressionThatCanNotBeMethodName	{can_be_command_ = 0;}
			)
		;

methodCall
{
	can_be_command_ = 1;
	boolean seen_arguments = false;
	boolean seen_block = false;
	boolean is_function = false;
}
		:	name:primaryExpressionThatCanBeMethodName	{is_function = (2 == can_be_command_);}
			(options{greedy=true;/*caused by command*/}:arg:methodInvocationArgumentWithParen	{seen_arguments = true;})?
			(options{greedy=true;/*caused by command*/}:codeBlock	{seen_block = true;})?
			{
				if (seen_arguments || seen_block || (is_function && (LA(1) == LBRACK_ARRAY_ACCESS)/*this is for a[1]*/))
				{
					#methodCall = #(#[CALL, "CALL"], #methodCall);
					can_be_command_ = 0;
				}
			}
		;

methodInvocationArgumentWithParen
		:	LPAREN!
				(methodInvocationArgumentWithoutParen[true])?
				(LINE_BREAK!)?
			RPAREN!
		;

methodInvocationArgumentWithoutParen[boolean should_ignore_line_break]
{
boolean seen_star_or_band = false;
}
		:	(normalMethodInvocationArgument[should_ignore_line_break]
			(options{greedy=true;/*caused by command*/}:
				COMMA!	{if (REST_ARG_PREFIX == LA(1) || BLOCK_ARG_PREFIX == LA(1))	{seen_star_or_band = true;break;}}
				normalMethodInvocationArgument[should_ignore_line_break]
			)*
			({seen_star_or_band}?	restMethodInvocationArgument
			|{seen_star_or_band}?	blockMethodInvocationArgument
			)?
			|	restMethodInvocationArgument
			|	blockMethodInvocationArgument
			)
			{#methodInvocationArgumentWithoutParen = #(#[ARG, "ARG"], #methodInvocationArgumentWithoutParen);}
		;

normalMethodInvocationArgument[boolean should_ignore_line_break]
		:	expression	(options{greedy=true;/*caused by command*/}:ASSOC^	expression)?	({should_ignore_line_break}?	LINE_BREAK!)?
		;

restMethodInvocationArgument
		:	REST_ARG_PREFIX	expression	(options{greedy=true;/*caused by command*/}:COMMA!	blockMethodInvocationArgument)?
		;

blockMethodInvocationArgument
		:	BLOCK_ARG_PREFIX	expression
		;

predefinedValue
		:	"nil"
		|	"true"
		|	"false"
		|	"__FILE__"
		|	"__LINE__"
		;

string
		:	DOUBLE_QUOTE_STRING
		|	SINGLE_QUOTE_STRING
		|	STRING_BEFORE_EXPRESSION_SUBSTITUTION^
			expression_substituation	{tellLexerWeHaveFinishedParsingStringExpressionSubstituation();}
			(STRING_BETWEEN_EXPRESSION_SUBSTITUTION	expression_substituation	{tellLexerWeHaveFinishedParsingStringExpressionSubstituation();})*
			STRING_AFTER_EXPRESSION_SUBSTITUTION
		;

protected
expression_substituation
		:	(	LCURLY_BLOCK!	(compoundStatement)?	RCURLY!
			|	GLOBAL_VARIABLE 
			|	INSTANCE_VARIABLE
			|	CLASS_VARIABLE
			)
		;

regex
		:	REGEX
		|	REGEX_BEFORE_EXPRESSION_SUBSTITUTION^
			expression_substituation	{tellLexerWeHaveFinishedParsingRegexExpressionSubstituation();}
			(STRING_BETWEEN_EXPRESSION_SUBSTITUTION	expression_substituation	{tellLexerWeHaveFinishedParsingRegexExpressionSubstituation();})*
			STRING_AFTER_EXPRESSION_SUBSTITUTION
		;

command_output
		:	COMMAND_OUTPUT
		|	COMMAND_OUTPUT_BEFORE_EXPRESSION_SUBSTITUTION^
			expression_substituation	{tellLexerWeHaveFinishedParsingStringExpressionSubstituation();}
			(STRING_BETWEEN_EXPRESSION_SUBSTITUTION	expression_substituation	{tellLexerWeHaveFinishedParsingStringExpressionSubstituation();})*
			STRING_AFTER_EXPRESSION_SUBSTITUTION
		;

heredoc
		:	HERE_DOC_BEGIN!
			(HERE_DOC_CONTENT
			|HERE_DOC_BEFORE_EXPRESSION_SUBSTITUTION^
			expression_substituation	{tellLexerWeHaveFinishedParsingHeredocExpressionSubstituation();}
			(HERE_DOC_BETWEEN_EXPRESSION_SUBSTITUTION	expression_substituation	{tellLexerWeHaveFinishedParsingHeredocExpressionSubstituation();})*
			HERE_DOC_AFTER_EXPRESSION_SUBSTITUTION
			)
		;

literal
		:	regex
		|	(options{greedy=true;/*caused by command*/}:string)+{ #literal = #([STRING, "STRING"], #literal); }
		|	heredoc
		|	command_output
		|	symbol
		|	W_ARRAY
		;

numeric
		:	INTEGER
		|	HEX		
		|	BINARY
		|	OCTAL
		|	FLOAT
		|	ASCII_VALUE
		;

primaryExpressionThatCanBeMethodName
		:	FUNCTION	{can_be_command_ = 2;}
		|	IDENTIFIER		//this is OK: p = 1; p 1
		|	"self"
		|	"super"		{can_be_command_ = 2;}
		|	CONSTANT
		|	LEADING_COLON2	CONSTANT
		|	"retry"
		|	"yield"		{can_be_command_ = 2;}
		|	keyword_defined
		|	"redo"
		|	EMPTY_ARRAY
		|	EMPTY_ARRAY_ACCESS
		|	UNARY_PLUS_MINUS_METHOD_NAME
		;

primaryExpressionThatCanNotBeMethodName
		:	INSTANCE_VARIABLE	
		|	GLOBAL_VARIABLE 
		|	CLASS_VARIABLE
		|	predefinedValue
		|	numeric
		|	literal
		|	arrayExpression
		|	hashExpression
		|	LPAREN^ compoundStatement RPAREN!
		|	ifExpression
		|	unlessExpression
		|	whileExpression			
		|	untilExpression			
		|	caseExpression		
		|	forExpression			
		|	exceptionHandlingExpression
		|	moduleDefination
		|	classDefination
		|	methodDefination
		;

primaryExpression
		:	primaryExpressionThatCanNotBeMethodName
		|	primaryExpressionThatCanBeMethodName	
		;

arrayReferenceArgument
		:	keyValuePair
			(	COMMA! {if (ASSIGN == LA(1)||RBRACK == LA(1)) break;}
				(seen_star:REST_ARG_PREFIX)?	keyValuePair	{if (null != seen_star) break;}
			)*
		|	REST_ARG_PREFIX	expression	(LINE_BREAK!)?
		;

arrayExpression
		:	LBRACK^
				(arrayReferenceArgument)?
			RBRACK!
		;

keyValuePair
		:	expression	(ASSOC	expression)?	(LINE_BREAK!)?
		;

hashExpression
		:	LCURLY_HASH^
				(
					keyValuePair
					(options{greedy=true;}:{RCURLY != LA(2)}?	COMMA!	keyValuePair)*
					(COMMA!)?
				)?
			RCURLY!
		;

bodyStatement
			:	(compoundStatement
				("rescue" exceptionList thenOrTermialOrColon (compoundStatement)?)*
				("else" (compoundStatement)?)?
				("ensure" (compoundStatement)?)?
			|	("rescue" exceptionList thenOrTermialOrColon (compoundStatement)?)+
				("else" (compoundStatement)?)?
				("ensure" (compoundStatement)?)?
			|	("ensure" (compoundStatement)?)
			)
			{#bodyStatement = #(#[BODY, "BODY"], #bodyStatement);}
		;

exceptionHandlingExpression
		:	"begin"^
			(bodyStatement)?
			"end"!
		;

exceptionList
		:	((className|INSTANCE_VARIABLE|IDENTIFIER)	(COMMA!	(className|INSTANCE_VARIABLE|IDENTIFIER))*)?	(ASSOC	variable)?
		;

ifExpression
		:	"if"^		(LINE_BREAK!)?
			expression thenOrTermialOrColon (compoundStatement)?
			("elsif" (expression)? thenOrTermialOrColon (compoundStatement)?)*
			("else" (compoundStatement)?)?
			"end"!
		;

unlessExpression
		:	"unless"^		(LINE_BREAK!)?
			expression thenOrTermialOrColon (compoundStatement)?
			("else" (compoundStatement)?)?
			"end"!
		;

caseExpression
		:	"case"^ (expression)? thenOrTermialOrColon		//(compoundStatement)?
			(keyword_when mrhs	thenOrTermialOrColon (compoundStatement)?)+
			("else" (compoundStatement)?)?
			"end"!
		;

forExpression
		:	"for"^	(LINE_BREAK!)?
			block_vars
			"in"		(LINE_BREAK!)?
			expression doOrTermialOrColon
			(compoundStatement)?
			"end"!
		;

whileExpression
		:	"while"^	(LINE_BREAK!)?
			expression doOrTermialOrColon
			(compoundStatement)?
			"end"!
		;

untilExpression
		:	"until"^	(LINE_BREAK!)?
			expression doOrTermialOrColon
			(compoundStatement)?
			"end"!
		;

moduleDefination
		:	"module"^	(LINE_BREAK!)?
			moduleName (options {greedy=true;}:terminal)?	{enterScope();}
			(bodyStatement)?
			"end"!			{leaveScope();}
		;

moduleName
		:	CONSTANT	(COLON2 CONSTANT)*
		;

classDefination
		:	"class"^	(LINE_BREAK!)?
			(	className	(LESS_THAN expression)?
			|	LEFT_SHIFT	expression
			)
			terminal			{enterScope();}
			(bodyStatement)?
			"end"!			{leaveScope();}
		;

/*
FIXME
cname	: tIDENTIFIER 
		| tCONSTANT
		;

cpath	: tCOLON3 cname 
		| cname 
		| primary_value tCOLON2 cname 
		;
*/
className
		:	(CONSTANT|FUNCTION)	(COLON2^	CONSTANT)*
		|	(LEADING_COLON2	CONSTANT)	(COLON2^	CONSTANT)*
		;

methodDefination
		:	"def"^		(options{greedy=true;}:LINE_BREAK!)?
			methodName	{enterScope();}
			methodDefinationArgument
			(bodyStatement)?
			"end"!		{leaveScope();}
		;

protected
methodNameSupplement
		:	(DOT|COLON2)
			(IDENTIFIER	(ASSIGN_WITH_NO_LEADING_SPACE)?
			|FUNCTION	(ASSIGN_WITH_NO_LEADING_SPACE)?
			|CONSTANT	(ASSIGN_WITH_NO_LEADING_SPACE)?
			|operatorAsMethodname
			|keyword
			)
		;

methodName
{boolean is_singleton_method = false;}
		:	(	operatorAsMethodname
			|	keywordAsMethodName
			|	(id:IDENTIFIER
				|function:FUNCTION	
				|constant:CONSTANT	
				)
				(methodNameSupplement{is_singleton_method = true;}|ASSIGN_WITH_NO_LEADING_SPACE)?
			|	LPAREN
				expression	(LINE_BREAK!)?
				RPAREN
				methodNameSupplement{is_singleton_method = true;}
			|	(INSTANCE_VARIABLE|CLASS_VARIABLE|GLOBAL_VARIABLE )	methodNameSupplement	{is_singleton_method = true;}
			|	("nil"|"self"|"true"|"false"|"__FILE__"|"__LINE__")	methodNameSupplement	{is_singleton_method = true;}
			|	UNARY_PLUS_MINUS_METHOD_NAME
			)
			{
				if (is_singleton_method) {
					#methodName = #(#[SINGLETON_METHOD, "SINGLETON_METHOD"], #methodName);
				}
			}
		;

operatorAsMethodname
		:	LEFT_SHIFT
		|	RIGHT_SHIFT
		|	EQUAL
		|	CASE_EQUAL
		|	GREATER_THAN
		|	GREATER_OR_EQUAL
		|	LESS_THAN
		|	LESS_OR_EQUAL
		|	PLUS
		|	MINUS
		|	STAR
		|	DIV
		|	MOD
		|	POWER
		|	BAND
		|	BOR
		|	BXOR
		|	(EMPTY_ARRAY	|EMPTY_ARRAY_ACCESS)	(options{greedy=true;}:ASSIGN_WITH_NO_LEADING_SPACE)?
		|	MATCH
		|	COMPARE
		|	BNOT
		|	SINGLE_QUOTE
		;

keyword
		:	keywordAsMethodName
		|	"nil"
		|	"self"
		|	"true"
		|	"false"
		|	"__FILE__"
		|	"__LINE__"
		;

keywordAsMethodName
		:	"and"
		|	"def"
		|	"end"
		|	"in"
		|	"or"
		|	"unless"
		|	"begin"
		|	"defined?"
		|	"ensure"
		|	"module"
		|	"redo"
		|	"super"
		|	"until"
		|	"BEGIN"
		|	"break"
		|	"do"
		|	"next"
		|	"rescue"
		|	"then"
		|	"when"
		|	"END"
		|	"case"
		|	"else"
		|	"for"
		|	"retry"
		|	"while"
		|	"alias"
		|	"class"
		|	"elsif"
		|	"if"
		|	"not"
		|	"return"
		|	"undef"
		|	"yield"
		;

methodDefinationArgument
		:	LPAREN!
				(methodDefinationArgumentWithoutParen)?
			RPAREN!	{tellLexerWeHaveFinishedParsingMethodparameters();}
			(options {greedy=true;}:terminal)?
		|	(methodDefinationArgumentWithoutParen)?	terminal
		;

methodDefinationArgumentWithoutParen
{boolean seen_star_or_band = false;}
		:	normalMethodDefinationArgument
				(COMMA!	{if (REST_ARG_PREFIX == LA(1) || BLOCK_ARG_PREFIX == LA(1))	{seen_star_or_band = true;break;}}	normalMethodDefinationArgument)*
				(	{seen_star_or_band}?	restMethodDefinationArgument
					|{seen_star_or_band}?	blockMethodDefinationArgument
				)?
		|	restMethodDefinationArgument
		|	blockMethodDefinationArgument
		;

normalMethodDefinationArgument
		:	variable	((ASSIGN|ASSIGN_WITH_NO_LEADING_SPACE)	expression)?
		;

variable
		:	id1:IDENTIFIER	{addVariable(id1);}
		|	id2:FUNCTION		{addVariable(id2);}
		;

restMethodDefinationArgument
		:	REST_ARG_PREFIX	(variable	(COMMA!	blockMethodDefinationArgument)?)?
		;

blockMethodDefinationArgument
		:	BLOCK_ARG_PREFIX	variable
		;


thenOrTermialOrColon
		:	"then"!
		|	terminal	("then"!)?
		|	COLON!
		;

doOrTermialOrColon
		:	DO_IN_CONDITION!
		|	terminal
		|	COLON!
		;

operator_ASSIGN				:	(ASSIGN|ASSIGN_WITH_NO_LEADING_SPACE)				(options{greedy=true;}:LINE_BREAK!)?;
/*
operator_QUESTION			:	QUESTION			(options{greedy=true;}:LINE_BREAK!)?;
operator_COLON				:	(COLON|COLON_WITH_NO_FOLLOWING_SPACE)			(options{greedy=true;}:LINE_BREAK!)?;
operator_PLUS_ASSIGN			:	PLUS_ASSIGN			(options{greedy=true;}:LINE_BREAK!)?;
operator_MINUS_ASSIGN		:	MINUS_ASSIGN		(options{greedy=true;}:LINE_BREAK!)?;
operator_STAR_ASSIGN		:	STAR_ASSIGN		(options{greedy=true;}:LINE_BREAK!)?;
operator_DIV_ASSIGN			:	DIV_ASSIGN			(options{greedy=true;}:LINE_BREAK!)?;
operator_MOD_ASSIGN			:	MOD_ASSIGN			(options{greedy=true;}:LINE_BREAK!)?;
operator_POWER_ASSIGN		:	POWER_ASSIGN		(options{greedy=true;}:LINE_BREAK!)?;
operator_BAND_ASSIGN		:	BAND_ASSIGN		(options{greedy=true;}:LINE_BREAK!)?;
operator_BXOR_ASSIGN		:	BXOR_ASSIGN		(options{greedy=true;}:LINE_BREAK!)?;
operator_BOR_ASSIGN			:	BOR_ASSIGN			(options{greedy=true;}:LINE_BREAK!)?;
operator_LEFT_SHIFT_ASSIGN	:	LEFT_SHIFT_ASSIGN	(options{greedy=true;}:LINE_BREAK!)?;
operator_RIGHT_SHIFT_ASSIGN	:	RIGHT_SHIFT_ASSIGN	(options{greedy=true;}:LINE_BREAK!)?;
{'fraction_non_alphanumeric': 0.14603049680275454, 'fraction_numerical': 0.00279390063944909, 'mean_word_length': 4.425491033304867, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 43, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4921994', 'n_tokens_mistral': 23425, 'n_tokens_neox': 20069, 'n_words': 2677}