<commit_before>globalvars = {}       # We will store the calculator's variables here
def lookup(map, name):
    #print "lookup", map, name
    for x, v in map:
        if x == name: return v
    if not globalvars.has_key(name):
        #print 'Undefined (defaulting to 0):', name
        pass
    return globalvars.get(name, 0)

%%
parser HSDL:
    #option:      "context-insensitive-scanner"

    token ENDTOKEN: " "
    token LPAREN: "("
    token RPAREN: ")"
    token TOKAMPERAMPERAMPER: "&&&"
    token TOKAMPERAMPER: "&&"
    token TOKAMPER: '&'
    token TOKBARBAR: "||"
    token TOKBAR: "|"
    token COLON: ':'
    token SEMICOLON: ';'
    token QUESTION: "?"
    token CARET: "^"
    token TOKTILDE: "~"
    token LESSLESS: "<<"
    token LEQ: "<="
    token LARROW: "<-"
    token LESS: "<"
    token GREATERGREATER: ">>"
    token GEQ: ">="
    token TOKGREATER: ">"
    token EQEQ: "=="
    token EQUAL: "="
    token STARSTAR: "**"
    token STAR: "*"
    token TOKNOTEQUAL: "!="
    token TOKEXCLAIM: "!"
    token TOKMINUS: "-"
    token TOKPERCENT: "%"
    token TOKPLUS: "+"
    token TOKSLASH: "/"
    token LBRACKET: "["
    token RBRACKET: "]"
    token LBRACE: "{"
    token RBRACE: "}"
    token APOSTROPHE: "'"
    token DOT: "."
    token COMMA: ','

    token TOKOPPLUS: '\\+'
    token TOKOPMINUS: '\\-'
    token TOKOPTILDECARET: '\\~^'
    token TOKOPCARETTILDE: '\\^~'
    token TOKOPCARET: '\\^'
    token TOKOPSLASH: '\\/'
    token TOKOPAMPERAMPER: '\\&&'
    token TOKOPAMPER: '\\&'
    token TOKOPBAR: '\\|'
    token TOKOPPERCENT: '\\%'
    token TOKOPLESSLESS: '\\<<'
    token TOKOPLEQ: '\\<='
    token TOKOPLESS: '\\<'
    token TOKOPGREATERGREATER: '\\>>'
    token TOKOPGEQ: '\\>='
    token TOKOPGREATER: '\\>'
    token TOKOPEQUALEQUAL: '\\=='
    token TOKOPSTARSTAR: '\\**'
    token TOKOPSTAR: '\\*'

    token TOKBDPI: '"BDPI"'
    token TOKBVI: '"BVI"'

    token NUM: " "
    token STR:   " "
    token VAR: " "
    token TYPEVAR: " "
    token CLASSVAR: " "
    token BUILTINVAR: " "

    token TOKACTIONSTATEMENT: "action"
    token TOKACTIONVALUESTATEMENT: "actionvalue"
    token TOKBEGIN: "begin"
    token TOKBITS: "Bits"
    token TOKBOUNDED: "Bounded"
    token TOKC: "C"
    token TOKCASE: "case"
    token TOKCF: "CF"
    token TOKCLOCKED_BY: "clocked_by"
    token TOKDEFAULT: "default"
    token TOKDEFAULT_CLOCK: "default_clock"
    token TOKDEFAULT_RESET: "default_reset"
    token TOKDEPENDENCIES: "dependencies"
    token TOKDERIVING: "deriving"
    token TOKDETERMINES: "determines"
    token TOKELSE: "else"
    token TOKENABLE: "enable"
    token TOKEND: "end"
    token TOKENDACTION: "endaction"
    token TOKENDACTIONVALUE: "endactionvalue"
    token TOKENDCASE: "endcase"
    token TOKENDFUNCTION: "endfunction"
    token TOKENDINSTANCE: "endinstance"
    token TOKENDINTERFACE: "endinterface"
    token TOKENDMETHOD: "endmethod"
    token TOKENDMODULE: "endmodule"
    token TOKENDPACKAGE: "endpackage"
    token TOKENDPAR: "endpar"
    token TOKENDRULE: "endrule"
    token TOKENDRULES: "endrules"
    token TOKENDSEQ: "endseq"
    token TOKENDTYPECLASS: "endtypeclass"
    token TOKENUM: "enum"
    token TOKEQ: "Eq"
    token TOKEXPORT: "export"
    token TOKFOR: "for"
    token TOKFUNCTION: "function"
    token TOKIF: "if"
    token TOKIMPORT: "import"
    #token TOKIN: "in"
    token TOKINPUT_CLOCK: "input_clock"
    token TOKINPUT_RESET: "input_reset"
    token TOKINSTANCE: "instance"
    token TOKINTERFACE: "interface"
    token TOKLET: "let"
    token TOKMATCH: "match"
    token TOKMATCHES: "matches"
    token TOKMETHOD: "method"
    token TOKMODULE: "module"
    token TOKNO_RESET: "no_reset"
    token TOKNUMERIC: "numeric"
    token TOKOUTPUT_CLOCK: "output_clock"
    token TOKOUTPUT_RESET: "output_reset"
    token TOKPACKAGE: "package"
    token TOKPAR: "par"
    token TOKPARAMETER: "parameter"
    token TOKPROVISOS: "provisos"
    token TOKREADY: "ready"
    token TOKRESET_BY: "reset_by"
    token TOKRETURN: "return"
    token TOKRULE: "rule"
    token TOKRULES: "rules"
    token TOKSB: "SB"
    token TOKSBR: "SBR"
    token TOKSCHEDULE: "schedule"
    token TOKSEQ: "seq"
    token TOKSTRUCT: "struct"
    token TOKTAGGED: "tagged"
    token TOKTYPE: "type"
    token TOKTYPECLASS: "typeclass"
    token TOKTYPEDEF: "typedef"
    token TOKUNION: "union"
    token TOKWHILE: "while"

############################################################################
############################# Datatypes ####################################
############################################################################

    rule type_decl:
        ( typevar_item_or_var
        | CLASSVAR typevar_item_or_var
        )
 [ LBRACKET NUM (COLON NUM)* RBRACKET ]

    rule formal_item:
        ( function_parameter
        | type_decl [ item_name ]
        )

    rule param_item:
        (   LPAREN param_item RPAREN
        |   ( [ TOKNUMERIC | TOKPARAMETER ] TOKTYPE VAR
            | formal_item
            | NUM
            | STR
            )
            [ VAR ]
            [ typevar_param_list ]
        )

    rule typevar_param_list:
        LPAREN [ param_item (COMMA param_item )* ] RPAREN

    rule typevar_item:
        TYPEVAR typevar_param_list

    rule typevar_item_or_var:
        VAR [ typevar_param_list ]
        | typevar_item

    rule Type_list:
        ( type_decl
        | LPAREN type_decl (COMMA type_decl)* RPAREN
        )

    rule enum_element:
        term<<[]>> [EQUAL expr<<[]>>]

    rule deriving_type_class:
        TOKEQ | TOKBITS | TOKBOUNDED

    rule single_type_definition:
        (   Type_list [ type_decl | TOKENABLE ]
        |   ( TOKENUM LBRACE enum_element [ ( ( COMMA enum_element )+ | SEMICOLON ) ]  RBRACE VAR
            | ( TOKSTRUCT | TOKUNION TOKTAGGED )
                LBRACE
                    ( single_type_definition )*
                RBRACE
                typevar_item_or_var
            )
            [ TOKDERIVING  LPAREN deriving_type_class ( COMMA deriving_type_class )* RPAREN ]
        )
        SEMICOLON

    rule formal_list:
        LPAREN [ formal_item ( COMMA formal_item)* ] RPAREN

    rule function_argument:
        ( typevar_item_or_var
        | LPAREN function_argument (COMMA function_argument)* RPAREN
        )
        [VAR [ formal_list ] ]

    rule function_parameter:
        TOKFUNCTION function_argument

############################################################################
############################# Expressions ##################################
############################################################################

    rule dot_item:
        DOT (VAR | STAR)

    rule dot_field_item:
        [ VAR COLON ] dot_item

    rule dot_field_list:
        dot_item
        | NUM
        | VAR
        | LBRACE dot_field_item (COMMA dot_field_item)* RBRACE

    rule tagged_dot_list:
        TOKTAGGED VAR
        [
            (dot_field_list
            | LPAREN
                ( dot_field_list
                | tagged_dot_list
                )
                RPAREN
            )
        ]

    rule matches_clause:
        ( tagged_dot_list
        | LPAREN tagged_dot_list RPAREN
        | LBRACE
             ( tagged_dot_list (COMMA tagged_dot_list)*
             | dot_field_item (COMMA dot_field_item)*
             ) RBRACE
        | TOKDEFAULT
        | NUM
        | VAR
        )

    rule expr<<V>>:
         exprint<<V>>
             [ QUESTION assign_rvalue COLON assign_rvalue ]
             {{return exprint}}

    # An expression is the sum and difference of factors
    rule exprint<<V>>:
        factor<<V>>         {{ n = factor }}
        (  TOKPLUS factor<<V>>
        |  TOKMINUS  factor<<V>>
        )* {{ return n }}

    # A factor is the product and division of terms
    rule factor<<V>>: nterm<<V>>           {{ v = nterm }}
        (  STAR nterm<<V>>
        |  STARSTAR nterm<<V>>
        |  APOSTROPHE nterm<<V>>
        |  TOKSLASH  nterm<<V>>
        |  CARET  nterm<<V>>
        |  LESS  nterm<<V>>
        |  TOKGREATER  nterm<<V>>
        |  GEQ  nterm<<V>>
        |  LESSLESS  nterm<<V>>
        #gets confused with assignment |  LEQ  nterm<<V>>
        |  GREATERGREATER  nterm<<V>>
        |  EQEQ  nterm<<V>>
        |  TOKNOTEQUAL  nterm<<V>>
        |  TOKAMPER  nterm<<V>>
        |  TOKAMPERAMPER  assign_rvalue
        |  TOKAMPERAMPERAMPER  nterm<<V>>
        |  TOKBAR  nterm<<V>>
        |  TOKBARBAR  nterm<<V>>
        |  TOKMATCHES matches_clause
        |  TOKPERCENT  nterm<<V>>
        )*  {{ return v }}

    rule nterm<<V>>:
        [ (TOKEXCLAIM | TOKTILDE | TOKMINUS | TOKBAR) ] term<<V>> {{ return term }}

    rule call_pitem:
        [ TOKCLOCKED_BY | TOKRESET_BY ] assign_rvalue

    rule call_params:
        LPAREN [ call_pitem ( COMMA call_pitem )* ] RPAREN

    rule item_name:
        ( VAR | TOKREADY | TOKENABLE)

    rule term_single:
        CLASSVAR* ( item_name | typevar_item )
        ( call_params
        | LBRACE [ VAR COLON assign_rvalue ( COMMA VAR COLON assign_rvalue )* ] RBRACE
        )*

    # A term is a number, variable, or an expression surrounded by parentheses
    rule term<<V>>:
        (    NUM
        |    (TOKTAGGED term_single)+ [term_single]
        |    BUILTINVAR [ param_list ]
        |    term_single
        |    STR {{ return STR }}
        |    paren_expression
        |    LBRACE assign_rvalue ( COMMA assign_rvalue )* RBRACE
        )
        (    LBRACKET expr<<V>> [ COLON expr<<V>> ] RBRACKET
        |    DOT VAR [ call_params ]
        )*

    rule assign_rvalue:
        ( statement_yielding_value
        | interface_declaration
        | TOKRULES [ COLON  VAR]
            ( rule_statement )*
            TOKENDRULES [ COLON  VAR]
        | function_parameter
        | function_operator [expr<<[]>>]
        | expr<<[]>> ( LEQ expr<<[]>> )* [expr<<[]>>]
        | QUESTION
        )
        # values like "(OP_Reg Reg_Normal data)" are allowed
        ( expr<<[]>> )*

    rule assign_opvalue:
        ( EQUAL | LEQ | LARROW ) assign_rvalue

    rule equal_value:
        EQUAL assign_rvalue

    rule function_operator:
        TOKOPPLUS | TOKOPMINUS | TOKOPSTAR
        | TOKOPCARET | TOKOPCARETTILDE | TOKOPTILDECARET
        | TOKOPSLASH
        | TOKOPAMPERAMPER | TOKOPAMPER
        | TOKOPBAR
        | TOKOPPERCENT
        | TOKOPLESS | TOKOPLESSLESS | TOKOPLEQ
        | TOKOPGREATER | TOKOPGREATERGREATER | TOKOPGEQ
        | TOKOPEQUALEQUAL
        | TOKOPSTARSTAR

    rule function_name:
        VAR | function_operator

    rule var_list_elem:
        VAR (DOT VAR)*

    rule var_list:
        LPAREN [var_list_elem (COMMA var_list_elem)*] RPAREN

    rule paren_expression:
        LPAREN [ assign_rvalue ] RPAREN

    rule param_list:
        LPAREN [ assign_rvalue (COMMA assign_rvalue)* ] RPAREN

    rule provisos_clause:
        TOKPROVISOS param_list

############################################################################
############################# Statements ###################################
############################################################################

    rule for_decl_item:
        # datatype might not be present
        type_decl [ VAR ] assign_opvalue

    rule for_variable_assignment:
        term<<[]>> assign_opvalue

    rule declared_item:
        term<<[]>> [ assign_opvalue ]

    rule rule_statement:
        TOKRULE VAR [paren_expression] SEMICOLON
        statement_list
        TOKENDRULE [ COLON  VAR ]

    rule match_arg:<commit_msg>added option for guarded rule<commit_after>globalvars = {}       # We will store the calculator's variables here
def lookup(map, name):
    #print "lookup", map, name
    for x, v in map:
        if x == name: return v
    if not globalvars.has_key(name):
        #print 'Undefined (defaulting to 0):', name
        pass
    return globalvars.get(name, 0)

%%
parser HSDL:
    #option:      "context-insensitive-scanner"

    token ENDTOKEN: " "
    token LPAREN: "("
    token RPAREN: ")"
    token TOKAMPERAMPERAMPER: "&&&"
    token TOKAMPERAMPER: "&&"
    token TOKAMPER: '&'
    token TOKBARBAR: "||"
    token TOKBAR: "|"
    token COLON: ':'
    token SEMICOLON: ';'
    token QUESTION: "?"
    token CARET: "^"
    token TOKTILDE: "~"
    token LESSLESS: "<<"
    token LEQ: "<="
    token LARROW: "<-"
    token LESS: "<"
    token GREATERGREATER: ">>"
    token GEQ: ">="
    token TOKGREATER: ">"
    token EQEQ: "=="
    token EQUAL: "="
    token STARSTAR: "**"
    token STAR: "*"
    token TOKNOTEQUAL: "!="
    token TOKEXCLAIM: "!"
    token TOKMINUS: "-"
    token TOKPERCENT: "%"
    token TOKPLUS: "+"
    token TOKSLASH: "/"
    token LBRACKET: "["
    token RBRACKET: "]"
    token LBRACE: "{"
    token RBRACE: "}"
    token APOSTROPHE: "'"
    token DOT: "."
    token COMMA: ','

    token TOKOPPLUS: '\\+'
    token TOKOPMINUS: '\\-'
    token TOKOPTILDECARET: '\\~^'
    token TOKOPCARETTILDE: '\\^~'
    token TOKOPCARET: '\\^'
    token TOKOPSLASH: '\\/'
    token TOKOPAMPERAMPER: '\\&&'
    token TOKOPAMPER: '\\&'
    token TOKOPBAR: '\\|'
    token TOKOPPERCENT: '\\%'
    token TOKOPLESSLESS: '\\<<'
    token TOKOPLEQ: '\\<='
    token TOKOPLESS: '\\<'
    token TOKOPGREATERGREATER: '\\>>'
    token TOKOPGEQ: '\\>='
    token TOKOPGREATER: '\\>'
    token TOKOPEQUALEQUAL: '\\=='
    token TOKOPSTARSTAR: '\\**'
    token TOKOPSTAR: '\\*'

    token TOKBDPI: '"BDPI"'
    token TOKBVI: '"BVI"'

    token NUM: " "
    token STR:   " "
    token VAR: " "
    token TYPEVAR: " "
    token CLASSVAR: " "
    token BUILTINVAR: " "

    token TOKACTIONSTATEMENT: "action"
    token TOKACTIONVALUESTATEMENT: "actionvalue"
    token TOKBEGIN: "begin"
    token TOKBITS: "Bits"
    token TOKBOUNDED: "Bounded"
    token TOKC: "C"
    token TOKCASE: "case"
    token TOKCF: "CF"
    token TOKCLOCKED_BY: "clocked_by"
    token TOKDEFAULT: "default"
    token TOKDEFAULT_CLOCK: "default_clock"
    token TOKDEFAULT_RESET: "default_reset"
    token TOKDEPENDENCIES: "dependencies"
    token TOKDERIVING: "deriving"
    token TOKDETERMINES: "determines"
    token TOKELSE: "else"
    token TOKENABLE: "enable"
    token TOKEND: "end"
    token TOKENDACTION: "endaction"
    token TOKENDACTIONVALUE: "endactionvalue"
    token TOKENDCASE: "endcase"
    token TOKENDFUNCTION: "endfunction"
    token TOKENDINSTANCE: "endinstance"
    token TOKENDINTERFACE: "endinterface"
    token TOKENDMETHOD: "endmethod"
    token TOKENDMODULE: "endmodule"
    token TOKENDPACKAGE: "endpackage"
    token TOKENDPAR: "endpar"
    token TOKENDRULE: "endrule"
    token TOKENDRULES: "endrules"
    token TOKENDSEQ: "endseq"
    token TOKENDTYPECLASS: "endtypeclass"
    token TOKENUM: "enum"
    token TOKEQ: "Eq"
    token TOKEXPORT: "export"
    token TOKFOR: "for"
    token TOKFUNCTION: "function"
    token TOKIF: "if"
    token TOKIMPORT: "import"
    #token TOKIN: "in"
    token TOKINPUT_CLOCK: "input_clock"
    token TOKINPUT_RESET: "input_reset"
    token TOKINSTANCE: "instance"
    token TOKINTERFACE: "interface"
    token TOKLET: "let"
    token TOKMATCH: "match"
    token TOKMATCHES: "matches"
    token TOKMETHOD: "method"
    token TOKMODULE: "module"
    token TOKNO_RESET: "no_reset"
    token TOKNUMERIC: "numeric"
    token TOKOUTPUT_CLOCK: "output_clock"
    token TOKOUTPUT_RESET: "output_reset"
    token TOKPACKAGE: "package"
    token TOKPAR: "par"
    token TOKPARAMETER: "parameter"
    token TOKPROVISOS: "provisos"
    token TOKREADY: "ready"
    token TOKRESET_BY: "reset_by"
    token TOKRETURN: "return"
    token TOKRULE: "rule"
    token TOKRULES: "rules"
    token TOKSB: "SB"
    token TOKSBR: "SBR"
    token TOKSCHEDULE: "schedule"
    token TOKSEQ: "seq"
    token TOKSTRUCT: "struct"
    token TOKTAGGED: "tagged"
    token TOKTYPE: "type"
    token TOKTYPECLASS: "typeclass"
    token TOKTYPEDEF: "typedef"
    token TOKUNION: "union"
    token TOKWHILE: "while"

############################################################################
############################# Datatypes ####################################
############################################################################

    rule type_decl:
        ( typevar_item_or_var
        | CLASSVAR typevar_item_or_var
        )
 [ LBRACKET NUM (COLON NUM)* RBRACKET ]

    rule formal_item:
        ( function_parameter
        | type_decl [ item_name ]
        )

    rule param_item:
        (   LPAREN param_item RPAREN
        |   ( [ TOKNUMERIC | TOKPARAMETER ] TOKTYPE VAR
            | formal_item
            | NUM
            | STR
            )
            [ VAR ]
            [ typevar_param_list ]
        )

    rule typevar_param_list:
        LPAREN [ param_item (COMMA param_item )* ] RPAREN

    rule typevar_item:
        TYPEVAR typevar_param_list

    rule typevar_item_or_var:
        VAR [ typevar_param_list ]
        | typevar_item

    rule Type_list:
        ( type_decl
        | LPAREN type_decl (COMMA type_decl)* RPAREN
        )

    rule enum_element:
        term<<[]>> [EQUAL expr<<[]>>]

    rule deriving_type_class:
        TOKEQ | TOKBITS | TOKBOUNDED

    rule single_type_definition:
        (   Type_list [ type_decl | TOKENABLE ]
        |   ( TOKENUM LBRACE enum_element [ ( ( COMMA enum_element )+ | SEMICOLON ) ]  RBRACE VAR
            | ( TOKSTRUCT | TOKUNION TOKTAGGED )
                LBRACE
                    ( single_type_definition )*
                RBRACE
                typevar_item_or_var
            )
            [ TOKDERIVING  LPAREN deriving_type_class ( COMMA deriving_type_class )* RPAREN ]
        )
        SEMICOLON

    rule formal_list:
        LPAREN [ formal_item ( COMMA formal_item)* ] RPAREN

    rule function_argument:
        ( typevar_item_or_var
        | LPAREN function_argument (COMMA function_argument)* RPAREN
        )
        [VAR [ formal_list ] ]

    rule function_parameter:
        TOKFUNCTION function_argument

############################################################################
############################# Expressions ##################################
############################################################################

    rule dot_item:
        DOT (VAR | STAR)

    rule dot_field_item:
        [ VAR COLON ] dot_item

    rule dot_field_list:
        dot_item
        | NUM
        | VAR
        | LBRACE dot_field_item (COMMA dot_field_item)* RBRACE

    rule tagged_dot_list:
        TOKTAGGED VAR
        [
            (dot_field_list
            | LPAREN
                ( dot_field_list
                | tagged_dot_list
                )
                RPAREN
            )
        ]

    rule matches_clause:
        ( tagged_dot_list
        | LPAREN tagged_dot_list RPAREN
        | LBRACE
             ( tagged_dot_list (COMMA tagged_dot_list)*
             | dot_field_item (COMMA dot_field_item)*
             ) RBRACE
        | TOKDEFAULT
        | NUM
        | VAR
        )

    rule expr<<V>>:
         exprint<<V>>
             [ QUESTION assign_rvalue COLON assign_rvalue ]
             {{return exprint}}

    # An expression is the sum and difference of factors
    rule exprint<<V>>:
        factor<<V>>         {{ n = factor }}
        (  TOKPLUS factor<<V>>
        |  TOKMINUS  factor<<V>>
        )* {{ return n }}

    # A factor is the product and division of terms
    rule factor<<V>>: nterm<<V>>           {{ v = nterm }}
        (  STAR nterm<<V>>
        |  STARSTAR nterm<<V>>
        |  APOSTROPHE nterm<<V>>
        |  TOKSLASH  nterm<<V>>
        |  CARET  nterm<<V>>
        |  LESS  nterm<<V>>
        |  TOKGREATER  nterm<<V>>
        |  GEQ  nterm<<V>>
        |  LESSLESS  nterm<<V>>
        #gets confused with assignment |  LEQ  nterm<<V>>
        |  GREATERGREATER  nterm<<V>>
        |  EQEQ  nterm<<V>>
        |  TOKNOTEQUAL  nterm<<V>>
        |  TOKAMPER  nterm<<V>>
        |  TOKAMPERAMPER  assign_rvalue
        |  TOKAMPERAMPERAMPER  nterm<<V>>
        |  TOKBAR  nterm<<V>>
        |  TOKBARBAR  nterm<<V>>
        |  TOKMATCHES matches_clause
        |  TOKPERCENT  nterm<<V>>
        )*  {{ return v }}

    rule nterm<<V>>:
        [ (TOKEXCLAIM | TOKTILDE | TOKMINUS | TOKBAR) ] term<<V>> {{ return term }}

    rule call_pitem:
        [ TOKCLOCKED_BY | TOKRESET_BY ] assign_rvalue

    rule call_params:
        LPAREN [ call_pitem ( COMMA call_pitem )* ] RPAREN

    rule item_name:
        ( VAR | TOKREADY | TOKENABLE)

    rule term_single:
        CLASSVAR* ( item_name | typevar_item )
        ( call_params
        | LBRACE [ VAR COLON assign_rvalue ( COMMA VAR COLON assign_rvalue )* ] RBRACE
        )*

    # A term is a number, variable, or an expression surrounded by parentheses
    rule term<<V>>:
        (    NUM
        |    (TOKTAGGED term_single)+ [term_single]
        |    BUILTINVAR [ param_list ]
        |    term_single
        |    STR {{ return STR }}
        |    paren_expression
        |    LBRACE assign_rvalue ( COMMA assign_rvalue )* RBRACE
        )
        (    LBRACKET expr<<V>> [ COLON expr<<V>> ] RBRACKET
        |    DOT VAR [ call_params ]
        )*

    rule assign_rvalue:
        ( statement_yielding_value
        | interface_declaration
        | TOKRULES [ COLON  VAR]
            ( rule_statement )*
            TOKENDRULES [ COLON  VAR]
        | function_parameter
        | function_operator [expr<<[]>>]
        | expr<<[]>> ( LEQ expr<<[]>> )* [expr<<[]>>]
        | QUESTION
        )
        # values like "(OP_Reg Reg_Normal data)" are allowed
        ( expr<<[]>> )*

    rule assign_opvalue:
        ( EQUAL | LEQ | LARROW ) assign_rvalue

    rule equal_value:
        EQUAL assign_rvalue

    rule function_operator:
        TOKOPPLUS | TOKOPMINUS | TOKOPSTAR
        | TOKOPCARET | TOKOPCARETTILDE | TOKOPTILDECARET
        | TOKOPSLASH
        | TOKOPAMPERAMPER | TOKOPAMPER
        | TOKOPBAR
        | TOKOPPERCENT
        | TOKOPLESS | TOKOPLESSLESS | TOKOPLEQ
        | TOKOPGREATER | TOKOPGREATERGREATER | TOKOPGEQ
        | TOKOPEQUALEQUAL
        | TOKOPSTARSTAR

    rule function_name:
        VAR | function_operator

    rule var_list_elem:
        VAR (DOT VAR)*

    rule var_list:
        LPAREN [var_list_elem (COMMA var_list_elem)*] RPAREN

    rule paren_expression:
        LPAREN [ assign_rvalue ] RPAREN

    rule param_list:
        LPAREN [ assign_rvalue (COMMA assign_rvalue)* ] RPAREN

    rule provisos_clause:
        TOKPROVISOS param_list

############################################################################
############################# Statements ###################################
############################################################################

    rule for_decl_item:
        # datatype might not be present
        type_decl [ VAR ] assign_opvalue

    rule for_variable_assignment:
        term<<[]>> assign_opvalue

    rule declared_item:
        term<<[]>> [ assign_opvalue ]

    rule rule_statement:
        TOKRULE VAR [paren_expression] [TOKIF paren_expression] SEMICOLON
        statement_list
        TOKENDRULE [ COLON  VAR ]

    rule match_arg:
{'fraction_non_alphanumeric': 0.1529345616539077, 'fraction_numerical': 0.00017300289779853813, 'mean_word_length': 2.3036148021145877, 'pattern_counts': {'":': 0, '<': 181, '<?xml version=': 0, '>': 179, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4563803', 'n_tokens_mistral': 8097, 'n_tokens_neox': 7707, 'n_words': 1683}