<commit_before>## Token

integer:
    (-|+)?[0-9]+

identifier:
    [a-zA-Z_][a-zA-Z_0-9]*

newline:
    "\r"
    "\n"

whitespace:
    "\r"
    "\n"
    "\t"
    "\v"
    "\f"
    " "

basicOperator:
    "+" "-" "*" "/"

logicBinaryOperator:
    "&&" "||" "==" "!="

logicUnaryOperator:
    "!"

compareOperator:
    ">" "<"

bitwiseBinaryOperator:
    "<<" ">>"      // shift
    "<<<" ">>>"    // rotate
    "^" "&" "|"


BinaryOperator:
    basicOperator
    logicBinaryOperator
    bitwiseBinaryOperator
    compareOperator

LogicOperator:
    logicBinaryOperator
    logicUnaryOperator

UnaryOperator:
    logicUnaryOperator

Attribute:
    identifier(".")

DeclBlock:
    StructDecl
    FuncDecl

StructDecl:
    "struct" identifier "{" VarDecl(newline) "}"

FuncDecl:
    "func" identifier "(" VarDecl(",") ")" VarDecl(",") "{" Statement(whitespace) "}"

Type:
    Attribute
    Type "[" "]"   // array type

VarDecl:
    ImplicitVarDecl
    ExplicitVarDecl

ImplicitVarDecl:
    identifier(",") Attribute                   // implicit assign to default value

ExplicitVarDecl:
    identifier(",") Type "=" Value(",")
    identifier(",") "=" Value(",")

Statement:
    VarDecl
    If
    While
    For
    Assignment

If:
    "if" Value "{" Statement(whitespace) "}"
    If "elif" value "{" Statement(whitespace) "}"
    If "else" "{" Statement(whitespace) "}"

While:
    "while" Value "{" Statement(whitespace) "}"

For:
    "for" ForHead; Value; ForTail "{" Statement(whitespace) "}"

ForHead:
    ExplicitVarDecl(",")

ForTail:
    Assignment(",)

Assignment:
    Attribute(,) "=" Value(",")

Value:
    Attribute
    FunctionCall
    "(" Value ")"
    Value binaryOperator Value
    

FunctionCall:
    Attribute "(" Value(",") ")"
    FunctionCall(".")
<commit_msg>Nearly fixed stage 1 syntax. Temp backup.<commit_after>## Token

integer:
    (-|+)?[0-9]+

identifier:
    [a-zA-Z_][a-zA-Z_0-9]*

newline:
    "\r"
    "\n"

whitespace:
    "\r"
    "\n"
    "\t"
    "\v"
    "\f"
    " "

basicOperator:
    "+" "-" "*" "/"

logicBinaryOperator:
    "&&" "||" "==" "!="

logicUnaryOperator:
    "!"

compareOperator:
    ">" "<"

bitwiseBinaryOperator:
    "<<" ">>"      // shift
    "<<<" ">>>"    // rotate
    "^" "&" "|"


BinaryOperator:
    basicOperator
    logicBinaryOperator
    bitwiseBinaryOperator
    compareOperator

LogicOperator:
    logicBinaryOperator
    logicUnaryOperator

UnaryOperator:
    logicUnaryOperator

Attribute:
    identifier(".")

DeclBlock:
    StructDecl
    FuncDecl

StructDecl:
    "struct" identifier "{" VarDecl(newline) "}"

FuncDecl:
    "func" identifier "(" VarDecl(",") ")" VarDecl(",") "{" Statement(whitespace) "}"

Type:
    Attribute
    Type "[" "]"   // array type

VarDecl:
    ImplicitVarDecl
    ExplicitVarDecl

ImplicitVarDecl:
    identifier(",") "=" Value(",")

ExplicitVarDecl:
    Type identifier(",") "=" Value(",")
    Type identifier(",")

AssignedVarDecl:
    Type identifier(",") "=" Value(",")
    identifier(",") "=" Value(",")

Statement:
    VarDecl
    If
    While
    For
    Assignment

If:
    "if" Value "{" Statement(whitespace) "}"
    If "elif" value "{" Statement(whitespace) "}"
    If "else" "{" Statement(whitespace) "}"

While:
    "while" Value "{" Statement(whitespace) "}"

For:
    "for" {ForHead} ";" Value ";" ForTail "{" Statement(whitespace) "}"

ForHead:
    AssignedVarDecl

ForTail:
    Assignment(",")

Assignment:
    Attribute(,) "=" Value(",")

Value:
    Attribute
    FunctionCall
    "(" Value ")"
    "{" Value(,) "}"
    Value binaryOperator Value
    UnaryOperator Value

FunctionCall:
    Attribute "(" Value(",") ")"
    FunctionCall(".")

{'fraction_non_alphanumeric': 0.185589519650655, 'fraction_numerical': 0.0024563318777292577, 'mean_word_length': 2.841719077568134, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5767637', 'n_tokens_mistral': 1301, 'n_tokens_neox': 1303, 'n_words': 238}