<commit_before>grammar MyScript;

parse
    : statement* EOF
    ;

statement
    : declarationStatement
    | expressionStatement
    ;

declarationStatement
    : localVariableDeclaration ';'
    ;

expressionStatement
    : statementExpression ';'
    ;

localVariableDeclaration
    : localVariableType localVariableDeclaratorList
    ;

localVariableType
    : VAR
    | type
    ;

localVariableDeclaratorList
    : localVariableDeclaratorList ',' localVariableDeclarator
    | localVariableDeclarator
    ;

localVariableDeclarator
    : IDENTIFIER
    | IDENTIFIER '=' expression
    ;

statementExpression
    : primaryExpression
    | assignmentExpression
    ;

expression
    : nonAssignmentExpression
    | assignmentExpression
    ;

assignmentExpression
    : primaryExpression '=' expression
    ;

nonAssignmentExpression
    : primaryExpression
    | chunkExpression
    | constrainExpression
    ;

primaryExpression
    : literal                                   # literalExpression
    | IDENTIFIER                                # lookupExpression
    | '(' expression ')'                        # parenthesizedExpression
    | primaryExpression '.' IDENTIFIER          # memberAccessExpression        
    | primaryExpression '(' argumentList? ')'   # invocationExpression
    ;

chunkExpression
    : 'chunk' tableIdent=IDENTIFIER 'by' 'column'? columnIdent=IDENTIFIER periodSpecifier? columnAggregateOperation*
    ;

periodSpecifier
    : 'per' period
    ;

period
    : singularTimeUnit
    | NUMBER pluralTimeUnit
    ;

singularTimeUnit
    : 'day'
    | 'week'
    | 'month'
    | 'year'
    ;

pluralTimeUnit
    : 'days'
    | 'weeks'
    | 'months'
    | 'years'
    ;

constrainExpression
    : 'constrain' tableIdent=IDENTIFIER 'where' conditionalExpression
    ;

conditionalExpression
    : '(' conditionalExpression ')'
    | conditionalExpression 'and' condition
    | conditionalExpression 'or' condition
    | condition
    ;

condition
    : columnIdent=IDENTIFIER comparisonOperator expression
    ;

comparisonOperator
    : '='
    | '<'
    | '<='
    | '>'
    | '>='
    ;

columnAggregateOperation
    : ',' aggregateOperation 'of' IDENTIFIER
    ;

aggregateOperation<commit_msg>Modified grammar for constraint expressions.<commit_after>grammar MyScript;

parse
    : statement* EOF
    ;

statement
    : declarationStatement
    | expressionStatement
    ;

declarationStatement
    : localVariableDeclaration ';'
    ;

expressionStatement
    : statementExpression ';'
    ;

localVariableDeclaration
    : localVariableType localVariableDeclaratorList
    ;

localVariableType
    : VAR
    | type
    ;

localVariableDeclaratorList
    : localVariableDeclaratorList ',' localVariableDeclarator
    | localVariableDeclarator
    ;

localVariableDeclarator
    : IDENTIFIER
    | IDENTIFIER '=' expression
    ;

statementExpression
    : primaryExpression
    | assignmentExpression
    ;

expression
    : nonAssignmentExpression
    | assignmentExpression
    ;

assignmentExpression
    : primaryExpression '=' expression
    ;

nonAssignmentExpression
    : primaryExpression
    | chunkExpression
    | constrainExpression
    ;

primaryExpression
    : literal                                   # literalExpression
    | IDENTIFIER                                # lookupExpression
    | '(' expression ')'                        # parenthesizedExpression
    | primaryExpression '.' IDENTIFIER          # memberAccessExpression        
    | primaryExpression '(' argumentList? ')'   # invocationExpression
    ;

chunkExpression
    : 'chunk' tableIdent=IDENTIFIER 'by' 'column'? columnIdent=IDENTIFIER periodSpecifier? columnAggregateOperation*
    ;

periodSpecifier
    : 'per' period
    ;

period
    : singularTimeUnit
    | NUMBER pluralTimeUnit
    ;

singularTimeUnit
    : 'day'
    | 'week'
    | 'month'
    | 'year'
    ;

pluralTimeUnit
    : 'days'
    | 'weeks'
    | 'months'
    | 'years'
    ;

constrainExpression
    : 'constrain' tableIdent=IDENTIFIER 'where' conditionalExpression
    ;

conditionalExpression
    : '(' conditionalExpression ')'
    | conditionalExpression booleanOperator condition
    | condition
    ;

booleanOperator
    : 'and'
    | 'or'
    ;

condition
    : columnIdent=IDENTIFIER comparisonOperator expression
    ;

comparisonOperator
    : '='
    | '<'
    | '<='
    | '>'
    | '>='
    ;

columnAggregateOperation
    : ',' aggregateOperation 'of' IDENTIFIER
    ;

aggregateOperation
{'fraction_non_alphanumeric': 0.08055925432756325, 'fraction_numerical': 0.0, 'mean_word_length': 2.6142742582197274, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1081290', 'n_tokens_mistral': 1248, 'n_tokens_neox': 1227, 'n_words': 205}