<commit_before>/*
  Converted to ANTLR 4 by <NAME> (Redline Smalltalk Project http://redline.st).
  Adapted from the Amber Smalltalk grammar perser.pegjs 
  
  2013/11/27 <NAME> (<EMAIL>)
*/

/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */

grammar Smalltalk;

script : sequence EOF;
sequence : ws temps? ws statements? ws;
ws : (SEPARATOR | COMMENT)*;
temps : PIPE (ws IDENTIFIER)* ws PIPE;
statements : answer
           | expressions ws PERIOD ws answer
           | expressions PERIOD?;
answer : CARROT ws expression ws PERIOD?;
expression : assignment | cascade | keywordSend | binarySend | primitive;
expressions : expression expressionList*;
expressionList : ws PERIOD ws expression;
cascade : ws (keywordSend | binarySend) (ws SEMI_COLON ws message ws)+;
message : binaryMessage | unaryMessage | keywordMessage;
assignment : variable ws ASSIGNMENT ws expression;
variable : IDENTIFIER;
binarySend : unarySend binaryTail?;
unarySend : operand ws unaryTail?;
keywordSend : binarySend keywordMessage;
keywordMessage : ws (keywordPair ws)+;
keywordPair : KEYWORD ws binarySend ws;
operand : literal | reference | subexpression;
subexpression : OPEN_PAREN ws expression ws CLOSE_PAREN;
literal : runtimeLiteral | parsetimeLiteral;
runtimeLiteral : dynamicDictionary | dynamicArray | block;
block : BLOCK_START blockParamList? sequence? ws BLOCK_END;
blockParamList : (ws BLOCK_PARAM)+ ws PIPE?;
dynamicDictionary : DYNDICT_START ws expressions? ws DYNARR_END;
dynamicArray : DYNARR_START ws expressions? ws DYNARR_END;
parsetimeLiteral : pseudoVariable | number | charConstant | literalArray | string | symbol;
number : numberExp | hex | stFloat | stInteger;
numberExp : (stFloat | stInteger) EXP stInteger;
charConstant : CHARACTER_CONSTANT;
hex : MINUS? HEX HEXDIGIT+;
stInteger : MINUS? DIGITS;
stFloat : MINUS? DIGITS PERIOD DIGITS;
pseudoVariable : RESERVED_WORD;
string : STRING;
symbol : HASH bareSymbol;
primitive : LT ws KEYWORD ws DIGITS ws GT;
bareSymbol : (IDENTIFIER | BINARY_SELECTOR) | KEYWORD+ | string;
literalArray : LITARR_START literalArrayRest;
literalArrayRest : ws ((parsetimeLiteral | bareLiteralArray | bareSymbol) ws)* CLOSE_PAREN;
bareLiteralArray : OPEN_PAREN literalArrayRest;
parseTimeLiteral : pseudoVariable | number | literalArray | string | symbol;
unaryTail : unaryMessage ws unaryTail? ws;
unaryMessage : ws unarySelector ~COLON;
unarySelector : IDENTIFIER;
keywords : KEYWORD+;
reference : variable;
binaryTail : binaryMessage binaryTail?;
binaryMessage : ws BINARY_SELECTOR ws (unarySend | operand);

PIPE : '|';
PERIOD : '.';
CARROT : '^';
COLON : ':';
SEMI_COLON : ';';
ASSIGNMENT : ':=';
MINUS : '-';
HASH : '#';<commit_msg>Fixed an error with the grammar parsing multiple expressions. Problem was in definition of UnaryMessage.<commit_after>/*
  Converted to ANTLR 4 by <NAME> (Redline Smalltalk Project http://redline.st).
  Adapted from the Amber Smalltalk grammar perser.pegjs 
  
  2013/11/27 <NAME> (<EMAIL>)
*/

grammar Smalltalk;

script : sequence EOF;
sequence : ws temps? ws statements? ws;
ws : (SEPARATOR | COMMENT)*;
temps : PIPE (ws IDENTIFIER)* ws PIPE;
statements : answer
           | expressions ws PERIOD ws answer
           | expressions PERIOD?;
answer : CARROT ws expression ws PERIOD?;
expression : assignment | cascade | keywordSend | binarySend | primitive;
expressions : expression expressionList*;
expressionList : ws PERIOD ws expression;
cascade : ws (keywordSend | binarySend) (ws SEMI_COLON ws message ws)+;
message : binaryMessage | unaryMessage | keywordMessage;
assignment : variable ws ASSIGNMENT ws expression;
variable : IDENTIFIER;
binarySend : unarySend binaryTail?;
unarySend : operand ws unaryTail?;
keywordSend : binarySend keywordMessage;
keywordMessage : ws (keywordPair ws)+;
keywordPair : KEYWORD ws binarySend ws;
operand : literal | reference | subexpression;
subexpression : OPEN_PAREN ws expression ws CLOSE_PAREN;
literal : runtimeLiteral | parsetimeLiteral;
runtimeLiteral : dynamicDictionary | dynamicArray | block;
block : BLOCK_START blockParamList? sequence? ws BLOCK_END;
blockParamList : (ws BLOCK_PARAM)+ ws PIPE?;
dynamicDictionary : DYNDICT_START ws expressions? ws DYNARR_END;
dynamicArray : DYNARR_START ws expressions? ws DYNARR_END;
parsetimeLiteral : pseudoVariable | number | charConstant | literalArray | string | symbol;
number : numberExp | hex | stFloat | stInteger;
numberExp : (stFloat | stInteger) EXP stInteger;
charConstant : CHARACTER_CONSTANT;
hex : MINUS? HEX HEXDIGIT+;
stInteger : MINUS? DIGITS;
stFloat : MINUS? DIGITS PERIOD DIGITS;
pseudoVariable : RESERVED_WORD;
string : STRING;
symbol : HASH bareSymbol;
primitive : LT ws KEYWORD ws DIGITS ws GT;
bareSymbol : (IDENTIFIER | BINARY_SELECTOR) | KEYWORD+ | string;
literalArray : LITARR_START literalArrayRest;
literalArrayRest : ws ((parsetimeLiteral | bareLiteralArray | bareSymbol) ws)* CLOSE_PAREN;
bareLiteralArray : OPEN_PAREN literalArrayRest;
parseTimeLiteral : pseudoVariable | number | literalArray | string | symbol;
unaryTail : unaryMessage ws unaryTail?;
unaryMessage : ws unarySelector;
unarySelector : IDENTIFIER;
keywords : KEYWORD+;
reference : variable;
binaryTail : binaryMessage binaryTail?;
binaryMessage : ws BINARY_SELECTOR ws (unarySend | operand);

PIPE : '|';
PERIOD : '.';
CARROT : '^';
COLON : ':';
SEMI_COLON : ';';
ASSIGNMENT : ':=';
MINUS : '-';
HASH : '#';
{'fraction_non_alphanumeric': 0.0924613987284287, 'fraction_numerical': 0.0032697547683923707, 'mean_word_length': 5.477647058823529, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3673711', 'n_tokens_mistral': 1818, 'n_tokens_neox': 1681, 'n_words': 542}