<commit_before>grammar ioke;

options { 
    output = AST; 
}

tokens {
    MESSAGE_SEND;
}

@lexer::header {
package ioke.lang.parser;
}

@header {
package ioke.lang.parser;
}

@lexer::members {
}

@members {
}

messageChain
    :
        commatedExpression+ EOF!
    ;

commatedExpression
    :
        assignmentExpression (Comma assignmentExpression)*
    ;

assignmentExpression
    :
        expression (Equals expression)?
    ;

expression
    :
        literal
    |   message
    |   Terminator
    ;

message
    :
        Identifier ('(' commatedExpression? ')')? -> ^(MESSAGE_SEND Identifier commatedExpression?)
    ;

literal
    :
        StringLiteral
    ;

Identifier
    :
        Letter (Letter|IDDigit|StrangeChars)*
    ;


StringLiteral
    :  '"' ( EscapeSequence | ~('\\'|'"') )* '"'
    ;

Terminator
    :
        ('\r'? '\n')+
    ;

Whitespace : Separator {skip();};

LineComment
    : '#' ~('\n'|'\r')* {$channel=HIDDEN;}
    ;

Equals
    :
        '='
    ;

Comma
    :
        ','
    ;

fragment
EscapeSequence
    :   '\\' ('b'|'t'|'n'|'f'|'r'|'\"'|'\''|'\\')
    |   UnicodeEscape
    |   OctalEscape
    ;

fragment
OctalEscape
    :   '\\' ('0'..'3') ('0'..'7') ('0'..'7')
    |   '\\' ('0'..'7') ('0'..'7')
    |   '\\' ('0'..'7')
    ;

fragment
UnicodeEscape
    :   '\\' 'u' HexDigit HexDigit HexDigit HexDigit
    ;

fragment
HexDigit : ('0'..'9'|'a'..'f'|'A'..'F') ;

fragment
Separator : (' ' | '\u000c' | '\u0009' | '\u000b' | '\\' '\u000a' )+ ;

fragment
StrangeChars
    :
        '_' |
        '!' |
        '?' |
        '<' |
        '>'
    ;

fragment
IDDigit
    :  '\u0030'..'\u0039' |
       '\u0660'..'\u0669' |
       '\u06f0'..'\u06f9' |
       '\u0966'..'\u096f' |
       '\u09e6'..'\u09ef' |
       '\u0a66'..'\u0a6f' |
       '\u0ae6'..'\u0aef' |
       '\u0b66'..'\u0b6f' |
       '\u0be7'..'\u0bef' |
       '\u0c66'..'\u0c6f' |
       '\u0ce6'..'\u0cef' |
       '\u0d66'..'\u0d6f' |
       '\u0e50'..'\u0e59' |
       '\u0ed0'..'\u0ed9' |
       '\u1040'..'\u1049'
   ;

fragment
Letter
    :  '\u0024'           |
       '\u0041'..'\u005a' |
       '\u005f'           |
       '\u0061'..'\u007a' |
       '\u00c0'..'\u00d6' |
       '\u00d8'..'\u00f6' |
       '\u00f8'..'\u00ff' |
       '\u0100'..'\u1fff' |
       '\u3040'..'\u318f' |
       '\u3300'..'\u337f' |
       '\u3400'..'\u3d2d' |
       '\u4e00'..'\u9fff' |
       '\uf900'..'\ufaff'
    ;
<commit_msg>Fix problems with earlier, so that arguments inside of parenthesis also works. Yay. Now to implement 'method'<commit_after>grammar ioke;

options { 
    output = AST; 
}

tokens {
    MESSAGE_SEND;
}

@lexer::header {
package ioke.lang.parser;
}

@header {
package ioke.lang.parser;
}

@lexer::members {
}

@members {
}

messageChain
    :
        assignmentExpression+ EOF!
    ;

commatedExpression
    :
        assignmentExpression+ (Comma assignmentExpression+)*
    ;

assignmentExpression
    :
        expression (Equals expression)?
    ;

expression
    :
        literal
    |   message
    |   Terminator
    ;

message
    :
        Identifier ('(' commatedExpression? ')')? -> ^(MESSAGE_SEND Identifier commatedExpression?)
    ;

literal
    :
        StringLiteral
    ;

Identifier
    :
        Letter (Letter|IDDigit|StrangeChars)*
    ;


StringLiteral
    :  '"' ( EscapeSequence | ~('\\'|'"') )* '"'
    ;

Terminator
    :
        ('\r'? '\n')+
    ;

Whitespace : Separator {skip();};

LineComment
    : '#' ~('\n'|'\r')* {$channel=HIDDEN;}
    ;

Equals
    :
        '='
    ;

Comma
    :
        ','
    ;

fragment
EscapeSequence
    :   '\\' ('b'|'t'|'n'|'f'|'r'|'\"'|'\''|'\\')
    |   UnicodeEscape
    |   OctalEscape
    ;

fragment
OctalEscape
    :   '\\' ('0'..'3') ('0'..'7') ('0'..'7')
    |   '\\' ('0'..'7') ('0'..'7')
    |   '\\' ('0'..'7')
    ;

fragment
UnicodeEscape
    :   '\\' 'u' HexDigit HexDigit HexDigit HexDigit
    ;

fragment
HexDigit : ('0'..'9'|'a'..'f'|'A'..'F') ;

fragment
Separator : (' ' | '\u000c' | '\u0009' | '\u000b' | '\\' '\u000a' )+ ;

fragment
StrangeChars
    :
        '_' |
        '!' |
        '?' |
        '<' |
        '>'
    ;

fragment
IDDigit
    :  '\u0030'..'\u0039' |
       '\u0660'..'\u0669' |
       '\u06f0'..'\u06f9' |
       '\u0966'..'\u096f' |
       '\u09e6'..'\u09ef' |
       '\u0a66'..'\u0a6f' |
       '\u0ae6'..'\u0aef' |
       '\u0b66'..'\u0b6f' |
       '\u0be7'..'\u0bef' |
       '\u0c66'..'\u0c6f' |
       '\u0ce6'..'\u0cef' |
       '\u0d66'..'\u0d6f' |
       '\u0e50'..'\u0e59' |
       '\u0ed0'..'\u0ed9' |
       '\u1040'..'\u1049'
   ;

fragment
Letter
    :  '\u0024'           |
       '\u0041'..'\u005a' |
       '\u005f'           |
       '\u0061'..'\u007a' |
       '\u00c0'..'\u00d6' |
       '\u00d8'..'\u00f6' |
       '\u00f8'..'\u00ff' |
       '\u0100'..'\u1fff' |
       '\u3040'..'\u318f' |
       '\u3300'..'\u337f' |
       '\u3400'..'\u3d2d' |
       '\u4e00'..'\u9fff' |
       '\uf900'..'\ufaff'
    ;

{'fraction_non_alphanumeric': 0.23360160965794768, 'fraction_numerical': 0.0716297786720322, 'mean_word_length': 1.617693522906793, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5749263', 'n_tokens_mistral': 2327, 'n_tokens_neox': 2163, 'n_words': 250}