<commit_before>grammar Babble;

file: expression (';'? expression)* EOF;

expression:
          PACKAGE name=ID packageBlock=block                 # packageExpression
          | IF test=expression THEN thenBlock=block
                               (ELSE elseBlock=block)?       # ifExpression
          | DEF name=ID (':' type)? ('=' value=expression)?  # defExpression
          | RETURN expression                                # returnExpression
          | WHILE test=expression THEN whileBlock=block      # whileExpression
          | expression '.' ID                                # selector
          | expression callParameters                        # call
          | block                                            # blockExpression
          | NOT expression                                   # booleanNot
          | left=expression op=('*' | '/') right=expression  # binaryOp
          | left=expression op=('+' | '-') right=expression  # binaryOp
          | left=expression op=('<' | '<=' | '=='
                       | '>=' | '>') right=expression        # binaryOp
          | left=expression op=AND right=expression          # booleanOp
          | left=expression op=OR right=expression           # booleanOp
          | functionType '->' functionBlock=block            # functionLiteral
          | name=ID '=' value=expression                     # assignExpression
          | NULL                                             # null
          | BOOLEAN                                          # boolean
          | RECURSE                                          # recurse
          | ID                                               # id
          | INT                                              # integer
          | FLOAT                                            # double
          | STRING                                           # string
          ;

block: '(' ')'
     | '(' expression (';'? expression)* ')';

parametersDeclaration: '(' parameterDeclaration (',' parameterDeclaration)* ')'
                     | '(' ')';

parameterDeclaration: ID (':' type)? ('=' defaultValue=expression)?;

callParameters: '(' callParameter (',' callParameter)* ')'
              | '(' ')';

callParameter: (ID ':')? expression;

type: simpleType
    | type parametersDeclaration
    | functionType
    ;

simpleType: ID ('.' ID)*;

functionType: parametersDeclaration ( ':' type | );

// Tokens
INT: [0-9]+;
FLOAT: [0-9]* '.' [0-9]+ ('E' [0-9]+)?;
STRING: '"' (~[\\"]|'\\\\'|'\\"')*? '"';
PLUS: '+';
MINUS: '-';
MUL: '*';
DIV: '/';
LT: '<';
LTE: '<=';
EQ: '==';
GTE: '>=';
GT: '>';
PACKAGE: 'package' | 'paquet';
IF: 'if' | 'si';
THEN: 'then' | 'alors';
ELSE: 'else' | 'sinon';
DEF: 'def' | 'soit';
RETURN: 'return' | 'retourne';
WHILE: 'while' | 'tant' WS* 'que';
AND: 'and' | 'et';
OR: 'or' | 'ou';<commit_msg>Explicit declaration of tokens.<commit_after>grammar Babble;

file: expression (';'? expression)* EOF;

expression:
          PACKAGE name=ID packageBlock=block                 # packageExpression
          | IF test=expression THEN thenBlock=block
                               (ELSE elseBlock=block)?       # ifExpression
          | DEF name=ID (':' type)? ('=' value=expression)?  # defExpression
          | RETURN expression                                # returnExpression
          | WHILE test=expression THEN whileBlock=block      # whileExpression
          | expression '.' ID                                # selector
          | expression callParameters                        # call
          | block                                            # blockExpression
          | NOT expression                                   # booleanNot
          | left=expression op=('*' | '/') right=expression  # binaryOp
          | left=expression op=('+' | '-') right=expression  # binaryOp
          | left=expression op=('<' | '<=' | '=='
                       | '>=' | '>') right=expression        # binaryOp
          | left=expression op=AND right=expression          # booleanOp
          | left=expression op=OR right=expression           # booleanOp
          | functionType '->' functionBlock=block            # functionLiteral
          | name=ID '=' value=expression                     # assignExpression
          | NULL                                             # null
          | BOOLEAN                                          # boolean
          | RECURSE                                          # recurse
          | ID                                               # id
          | INT                                              # integer
          | FLOAT                                            # double
          | STRING                                           # string
          ;

block: '(' ')'
     | '(' expression (';'? expression)* ')';

parametersDeclaration: '(' parameterDeclaration (',' parameterDeclaration)* ')'
                     | '(' ')';

parameterDeclaration: ID (':' type)? ('=' defaultValue=expression)?;

callParameters: '(' callParameter (',' callParameter)* ')'
              | '(' ')';

callParameter: (ID ':')? expression;

type: simpleType
    | type parametersDeclaration
    | functionType
    ;

simpleType: ID ('.' ID)*;

functionType: parametersDeclaration ( ':' type | );

// Tokens
INT: [0-9]+;
FLOAT: [0-9]* '.' [0-9]+ ('E' [0-9]+)?;
STRING: '"' (~[\\"]|'\\\\'|'\\"')*? '"';
PLUS: '+';
MINUS: '-';
MUL: '*';
DIV: '/';
LT: '<';
LTE: '<=';
EQ: '==';
GTE: '>=';
GT: '>';
COLON: ':';
ASSIGN: '=';
DOT: '.';
ARROW: '->';
RPAREN: '(';
LPAREN: ')';
COMMA: ',';
PACKAGE: 'package' | 'paquet';
IF: 'if' | 'si';
THEN: 'then' | 'alors';
ELSE: 'else' | 'sinon';
DEF: 'def' | 'soit';
RETURN: 'return' | 'retourne';
WHILE: 'while' | 'tant' WS* 'que';
AND: 'and' | 'et';
OR: 'or' | 'ou';
{'fraction_non_alphanumeric': 0.15683229813664595, 'fraction_numerical': 0.0027605244996549345, 'mean_word_length': 1.3924886504333471, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 38, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7313033', 'n_tokens_mistral': 1508, 'n_tokens_neox': 1477, 'n_words': 321}