<commit_before>grammar Aql;

options {
	output=AST;
}

@parser::header {package annis.ql;}
@lexer::header {package annis.ql;}

start 
	: (expr)+;

ID  :	('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'0'..'9'|'_')*
    ;

fragment
HEX_DIGIT : ('0'..'9'|'a'..'f'|'A'..'F') ;

fragment
UNICODE_ESC
    :   '\\' 'u' HEX_DIGIT HEX_DIGIT HEX_DIGIT HEX_DIGIT
    ;

fragment
ESC_SEQ
    :   '\\' ('b'|'t'|'n'|'f'|'r'|'\"'|'\''|'\\')
    |   UNICODE_ESC
    |   OCTAL_ESC
    ;

fragment
OCTAL_ESC
    :   '\\' ('0'..'3') ('0'..'7') ('0'..'7')
    |   '\\' ('0'..'7') ('0'..'7')
    |   '\\' ('0'..'7')
    ;
    
text_spec 
	:	'"'^ ( ESC_SEQ | ~('"') )* '"'!
	|	'/'^ ( ESC_SEQ | ~('/') )* '/'!
	;

qName
	: namespace+=ID ':' name+=ID -> ^($name $namespace)
	| ID^
	;

expr 
	: qName
	|	qName '=' text_spec -> ^('=' qName text_spec)
	| qName '!=' text_spec -> ^('=' qName text_spec)
	|	'(' expr+ ')' -> ^('(' expr+)
	|	'&'^ expr
	| '|'^ expr
	;
	
	<commit_msg>refactoring the grammar (not finished yet)<commit_after>grammar Aql;

options {
	output=AST;
	backtrack=true;
}

@parser::header {package annis.ql;}
@lexer::header {package annis.ql;}

start 
	: andExpr
	| orExpr
	;

ID  :	('a'..'z'|'A'..'Z'|'_') ('a'..'z'|'A'..'Z'|'0'..'9'|'_')*
    ;

fragment
HEX_DIGIT : ('0'..'9'|'a'..'f'|'A'..'F') ;

fragment
UNICODE_ESC
    :   '\\' 'u' HEX_DIGIT HEX_DIGIT HEX_DIGIT HEX_DIGIT
    ;

fragment
ESC_SEQ
    :   '\\' ('b'|'t'|'n'|'f'|'r'|'\"'|'\''|'\\')
    |   UNICODE_ESC
    |   OCTAL_ESC
    ;

fragment
OCTAL_ESC
    :   '\\' ('0'..'3') ('0'..'7') ('0'..'7')
    |   '\\' ('0'..'7') ('0'..'7')
    |   '\\' ('0'..'7')
    ;
    
text_spec 
	:	'"'^ ( ESC_SEQ | ~('"') )* '"'!
	|	'/'^ ( ESC_SEQ | ~('/') )* '/'!
	;

qName
	: namespace=ID ':' name=ID -> ^($name $namespace)
	| ID^
	;

andExpr
	: (expr '&')+ expr -> ^('&' expr+ expr)
	;
	
orExpr
	: (expr '|')+ expr -> ^('&' expr+ expr)
	;	

expr 
	: qName
	|	qName '=' text_spec -> ^('=' qName text_spec)
	| qName '!=' text_spec -> ^('=' qName text_spec)
	|	'('! andExpr^ ')'!
	|	'('! orExpr^ ')'!
	;
	
	
{'fraction_non_alphanumeric': 0.3251833740831296, 'fraction_numerical': 0.015647921760391197, 'mean_word_length': 2.576923076923077, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7148356', 'n_tokens_mistral': 1114, 'n_tokens_neox': 1067, 'n_words': 137}