<commit_before>grammar CollectionQuery;

@header {package de.ids_mannheim.korap.query.serialize.util;}

/*
 -- author: jbingel
 -- date: 14-05-11
*/

/*
 * LEXER SECTION
 */
LRB					: '(';
RRB					: ')';
LT					: '<';
GT					: '>';
LEQ					: '<=';
GEQ					: '>=';
EQ					: '=';
NE					: '!=';
AND					: '&' | 'AND' | 'and' | 'UND' | 'und' ;
OR					: '|' | 'OR' | 'or' | 'ODER' | 'oder' ;
QMARK				: '?';
SLASH				: '/';
WS 					: ( ' ' | '\t' | '\r' | '\n' )+ -> skip ;
fragment NO_RE      : ~[ \t\/];
fragment ALPHABET   : ~('\t' | ' ' | '/' | '*' | '?' | '+' | '{' | '}' | '[' | ']'
                    | '(' | ')' | '|' | '"' | ',' | ':' | '\'' | '\\' | '!' | '=' | '~' | '&' | '^' | '<' | '>' );
NUMBER              : [0-9]+;

NL                  : [\r\n] -> skip;
ws                  : WS+;

WORD                : ALPHABET+;

/*
 * Regular expressions
 */
fragment FOCC	     : '{' WS* ( [0-9]* WS* ',' WS* [0-9]+ | [0-9]+ WS* ','? ) WS* '}';
fragment RE_char     : ~('*' | '?' | '+' | '{' | '}' | '[' | ']' | '/'
         	            | '(' | ')' | '|' | '"' | ':' | '\'' | '\\');
fragment RE_alter    : ((RE_char | ('(' REGEX ')') | RE_chgroup) '|' REGEX )+;
fragment RE_chgroup  : '[' RE_char+ ']';
fragment RE_quant	 : (RE_star | RE_plus | RE_occ) QMARK?;
fragment RE_opt      : (RE_char | RE_chgroup | ( '(' REGEX ')')) '?';
fragment RE_star     : (RE_char | RE_chgroup | ( '(' REGEX ')')) '*';
fragment RE_plus     : (RE_char | RE_chgroup | ( '(' REGEX ')')) '+';
fragment RE_occ      : (RE_char | RE_chgroup | ( '(' REGEX ')')) FOCC;
fragment RE_group    : '(' REGEX ')';
REGEX     		     : SLASH ('.' | RE_char | RE_alter | RE_chgroup | RE_opt | RE_quant | RE_group)* SLASH;

/*
 * PARSER SECTION
 */

regex
:	REGEX
;

conj
:	AND | OR;

operator
:	EQ | NE | LT | GT | LEQ | GEQ;

expr
:	(value operator)? field operator value
;
	
field
:	WORD
;
	
value
: WORD 
| NUMBER 
| '"' (WORD ws*)+'"'
| regex
;

/*	
andGroup
: 	(((LRB exprGroup RRB)|expr) AND)+ ((LRB exprGroup RRB)|expr)
;

orGroup
: 	(((LRB exprGroup RRB)|expr) OR)+ ((LRB exprGroup RRB)|expr)
;
*/

relation
:	(expr|exprGroup) conj (expr|exprGroup|relation)
; 

exprGroup
:	LRB (expr | exprGroup | relation) RRB
;

start
:	( expr 
	| exprGroup EOF 
	| relation EOF ) 
;<commit_msg>removed EOF operator to allow embedding in parentheses<commit_after>grammar CollectionQuery;

@header {package de.ids_mannheim.korap.query.serialize.util;}

/*
 -- author: jbingel
 -- date: 14-05-11
*/

/*
 * LEXER SECTION
 */
LRB					: '(';
RRB					: ')';
LT					: '<';
GT					: '>';
LEQ					: '<=';
GEQ					: '>=';
EQ					: '=';
NE					: '!=';
AND					: '&' | 'AND' | 'and' | 'UND' | 'und' ;
OR					: '|' | 'OR' | 'or' | 'ODER' | 'oder' ;
QMARK				: '?';
SLASH				: '/';
WS 					: ( ' ' | '\t' | '\r' | '\n' )+ -> skip ;
fragment NO_RE      : ~[ \t\/];
fragment ALPHABET   : ~('\t' | ' ' | '/' | '*' | '?' | '+' | '{' | '}' | '[' | ']'
                    | '(' | ')' | '|' | '"' | ',' | ':' | '\'' | '\\' | '!' | '=' | '~' | '&' | '^' | '<' | '>' );
NUMBER              : [0-9]+;

NL                  : [\r\n] -> skip;
ws                  : WS+;

WORD                : ALPHABET+;

/*
 * Regular expressions
 */
fragment FOCC	     : '{' WS* ( [0-9]* WS* ',' WS* [0-9]+ | [0-9]+ WS* ','? ) WS* '}';
fragment RE_char     : ~('*' | '?' | '+' | '{' | '}' | '[' | ']' | '/'
         	            | '(' | ')' | '|' | '"' | ':' | '\'' | '\\');
fragment RE_alter    : ((RE_char | ('(' REGEX ')') | RE_chgroup) '|' REGEX )+;
fragment RE_chgroup  : '[' RE_char+ ']';
fragment RE_quant	 : (RE_star | RE_plus | RE_occ) QMARK?;
fragment RE_opt      : (RE_char | RE_chgroup | ( '(' REGEX ')')) '?';
fragment RE_star     : (RE_char | RE_chgroup | ( '(' REGEX ')')) '*';
fragment RE_plus     : (RE_char | RE_chgroup | ( '(' REGEX ')')) '+';
fragment RE_occ      : (RE_char | RE_chgroup | ( '(' REGEX ')')) FOCC;
fragment RE_group    : '(' REGEX ')';
REGEX     		     : SLASH ('.' | RE_char | RE_alter | RE_chgroup | RE_opt | RE_quant | RE_group)* SLASH;

/*
 * PARSER SECTION
 */

regex
:	REGEX
;

conj
:	AND | OR;

operator
:	EQ | NE | LT | GT | LEQ | GEQ;

expr
:	(value operator)? field operator value
//|	LRB expr RRB
;
	
field
:	WORD
;
	
value
: WORD 
| NUMBER 
| '"' (WORD ws*)+'"'
| regex
;


relation
:	(expr|exprGroup) conj (expr|exprGroup|relation)
//|	LRB relation RRB
; 

exprGroup
:	LRB (expr | exprGroup | relation) RRB
;

start
:	( expr 
	| exprGroup 
	| relation  ) 
;
{'fraction_non_alphanumeric': 0.26220614828209765, 'fraction_numerical': 0.006329113924050633, 'mean_word_length': 2.1494661921708187, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5196191', 'n_tokens_mistral': 2008, 'n_tokens_neox': 1900, 'n_words': 341}