<commit_before>#! nqp
# Copyright (C) 2009-2010, Parrot Foundation.
# $Id$

INIT { pir::load_bytecode('HLL.pbc'); }

grammar Ops::Compiler::Grammar is HLL::Grammar;

rule TOP {
    <body>
    [ $ || <.panic: 'Syntax error'> ]
}

rule body {
    [ <preamble> | <op> ]*
}

token preamble {
    <begin_preamble>
    <preamble_guts>
    <end_preamble>
}

regex preamble_guts {
    .*? <?end_preamble>
}

token begin_preamble {
    ^^'BEGIN_OPS_PREAMBLE'
}

token end_preamble {
    ^^'END_OPS_PREAMBLE'
}

rule op {
    <op_type>? 'op' <op_name=identifier>
    [ '(' <signature> ')' || <.panic: "Fail to parse signature"> ]
    <op_flag>*
    [ <op_body> || <.panic: "Fail to parse op body"> ]
    {*}
}

token op_type {
    [ 'inline' | 'function' ]
}

rule signature { [ [<.ws><op_param><.ws>] ** ',' ]? }

rule op_param {
    <op_param_direction> <op_param_type>
}

token op_param_direction {
    # Order is crucial. NQP doesn't support LTM yet.
    [
    | 'inout'
    | 'inconst'
    | 'invar'
    | 'in'
    | 'out'
    ]
}

token op_param_type {
    # Order is crucial. NQP doesn't support LTM yet.
    [
    | 'INTKEY'
    | 'INT'
    | 'NUM'
    | 'STR'
    | 'PMC'
    | 'KEY'
    | 'LABEL'
    ]
}

rule op_flag {
    ':' <identifier>
}

# OpBody starts with '{' and ends with single '}' on line.
regex op_body {
    '{'
    <body_word>*?
    ^^ '}'
}

#Process op body by breaking it into "words" consisting entirely of whitespace,
#alnums or a single punctuation, then checking for interesting macros (e.g $1
#or goto NEXT() ) in the midst of the words.
token body_word {
    [
    || <.ws> <macro_param> <.ws>
    || <.ws> <op_macro> <.ws>
    || $<word>=[ [<alnum>+|<punct>]? <ws> ]
    ]
}

token macro_param {
    '$' $<num>=<[1..9]> # Up to nine params.
}

rule op_macro {
    <macro_type> <macro_destination> '(' <body_word>*? ')'
}

token macro_type {
    [
    | 'goto'
    | 'expr'
    | 'restart'
    ]
}

token macro_destination {
    [
    | 'OFFSET'
    | 'ADDRESS'
    | 'NEXT'
    ]
}

token identifier {
    <.ident>
}

# ws handles whitespace, pod and perl and C comments
token ws {
  [
  | \s+
  | '#' \N*
  | ^^ '=' .*? \n '=cut'
  | '/*' .*? '*/'
  ]*
}


# vim: expandtab shiftwidth=4 ft=perl6:
<commit_msg>Speed up opsc grammar more by using more specific tokens instead of "catch-them-all" <alnum>|<punct><commit_after>#! nqp
# Copyright (C) 2009-2010, Parrot Foundation.
# $Id$

INIT { pir::load_bytecode('HLL.pbc'); }

grammar Ops::Compiler::Grammar is HLL::Grammar;

rule TOP {
    <body>
    [ $ || <.panic: 'Syntax error'> ]
}

rule body {
    [ <preamble> | <op> ]*
}

token preamble {
    <begin_preamble>
    <preamble_guts>
    <end_preamble>
}

regex preamble_guts {
    .*? <?end_preamble>
}

token begin_preamble {
    ^^'BEGIN_OPS_PREAMBLE'
}

token end_preamble {
    ^^'END_OPS_PREAMBLE'
}

rule op {
    <op_type>? 'op' <op_name=identifier>
    [ '(' <signature> ')' || <.panic: "Fail to parse signature"> ]
    <op_flag>*
    [ <op_body> || <.panic: "Fail to parse op body"> ]
    {*}
}

token op_type {
    [ 'inline' | 'function' ]
}

rule signature { [ [<.ws><op_param><.ws>] ** ',' ]? }

rule op_param {
    <op_param_direction> <op_param_type>
}

token op_param_direction {
    # Order is crucial. NQP doesn't support LTM yet.
    [
    | 'inout'
    | 'inconst'
    | 'invar'
    | 'in'
    | 'out'
    ]
}

token op_param_type {
    # Order is crucial. NQP doesn't support LTM yet.
    [
    | 'INTKEY'
    | 'INT'
    | 'NUM'
    | 'STR'
    | 'PMC'
    | 'KEY'
    | 'LABEL'
    ]
}

rule op_flag {
    ':' <identifier>
}

# OpBody starts with '{' and ends with single '}' on line.
regex op_body {
    '{'
    <body_word>*?
    ^^ '}'
}

#Process op body by breaking it into "words" consisting entirely of whitespace,
#alnums or a single punctuation, then checking for interesting macros (e.g $1
#or goto NEXT() ) in the midst of the words.
token body_word {
    [
    || <.ws> <macro_param> <.ws>
    || <.ws> <op_macro> <.ws>
    || <word>
    ]
}

token word {
    || <quote>
    || <ident>
    || <raw_word> <ws>
    || <ws>
}

token raw_word {
    <alnum>|<punct>
}

proto token quote { <...> }
token quote:sym<apos> { <?[']> <quote_EXPR: ':q'>  }
token quote:sym<dblq> { <?["]> <quote_EXPR: ':q'> }

token macro_param {
    '$' $<num>=<[1..9]> # Up to nine params.
}

rule op_macro {
    <macro_type> <macro_destination> '(' <body_word>*? ')'
}

token macro_type {
    [
    | 'goto'
    | 'expr'
    | 'restart'
    ]
}

token macro_destination {
    [
    | 'OFFSET'
    | 'ADDRESS'
    | 'NEXT'
    ]
}

token identifier {
    <.ident>
}

# ws handles whitespace, pod and perl and C comments
token ws {
  [
  | \s+
  | '#' \N*
  | ^^ '=' .*? \n '=cut'
  | '/*' .*? '*/'
  ]*
}


# vim: expandtab shiftwidth=4 ft=perl6:

{'fraction_non_alphanumeric': 0.18554116172168825, 'fraction_numerical': 0.005432511491851233, 'mean_word_length': 2.599248120300752, 'pattern_counts': {'":': 0, '<': 92, '<?xml version=': 0, '>': 92, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3821055', 'n_tokens_mistral': 2052, 'n_tokens_neox': 1967, 'n_words': 438}