<commit_before>grammar Regex::P6Regex::Grammar is PCT::Grammar;

    token ws { [ \s+ | '#' \N*\n ]* }

    token TOP {
        <nibbler>
        [ <.ws> $ || <.panic: "Syntax error"> ]
        {*}
    }

    rule nibbler {
        ['||'|'|'|'&&'|'&']?
        <termish> 
        [ ['||'|'|'] 
          [ <termish> || <.panic: "Null pattern not allowed"> ]
        ]*
        {*}
    }

    token termish {
        <.ws>
        <noun=quantified_atom>+
        {*}
    }

    token quantified_atom {
        <atom>
	<.ws>
        [ <quantifier> <.ws> ]?
        {*}
    }

    token atom {
        :dba('regex atom')
        [
        | \w [ \w+! <?before \w> ]?
        | <metachar>
        ]
        {*}
    }

    # proto token quantifier { <...> }
    token quantifier:sym<*> { $<sym>:=['*'] <quantmod> {*} }
    token quantifier:sym<+> { $<sym>:=['+'] <quantmod> {*} }
    token quantifier:sym<?> { $<sym>:=['?'] <quantmod> {*} }

    token quantmod { ':'? [ '?' | '!' | '+' ]? {*} }

    # proto token metachar { <...> }
    token metachar:sym<[ ]> { '[' <nibbler> ']' {*} }
    token metachar:sym<.> { $<sym>:=['.'] {*} }
    token metachar:sym<^> { $<sym>:=['^'] {*} }
    token metachar:sym<^^> { $<sym>:=['^^'] {*} }
    token metachar:sym<$> { $<sym>:=['$'] {*} }
    token metachar:sym<$$> { $<sym>:=['$$'] {*} }
    token metachar:sym<lwb> { $<sym>:=['<<'|'«'] {*} }
    token metachar:sym<rwb> { $<sym>:=['>>'|'»'] {*} }
    token metachar:sym<bs> { \\ <backslash> {*} }
    token metachar:sym<assert> { 
        '<' <assertion> 
        [ '>' || <.panic: "regex assertion not terminated by angle bracket"> ]
        {*}
    }

    # proto token backslash { <...> }
    token backslash:sym<w> { $<sym>:=[<[dswnDSWN]>] {*} }
    token backslash:sym<b> { $<sym>:=[<[bB]>] {*} }
    token backslash:sym<e> { $<sym>:=[<[eE]>] {*} }
    token backslash:sym<f> { $<sym>:=[<[fF]>] {*} }
    token backslash:sym<h> { $<sym>:=[<[hH]>] {*} }
    token backslash:sym<r> { $<sym>:=[<[rR]>] {*} }
    token backslash:sym<t> { $<sym>:=[<[tT]>] {*} }
    token backslash:sym<v> { $<sym>:=[<[vV]>] {*} }
    token backslash:sym<A> { 'A' <.obs: '\\A as beginning-of-string matcher;^'> }
    token backslash:sym<z> { 'z' <.obs: '\\z as end-of-string matcher;$'> }
    token backslash:sym<Z> { 'Z' <.obs: '\\Z as end-of-string matcher;\\n?$'> }<commit_msg>Convert := operators to = in P6Regex/Grammar.pm .<commit_after>grammar Regex::P6Regex::Grammar is PCT::Grammar;

    token ws { [ \s+ | '#' \N*\n ]* }

    token TOP {
        <nibbler>
        [ <.ws> $ || <.panic: "Syntax error"> ]
        {*}
    }

    rule nibbler {
        ['||'|'|'|'&&'|'&']?
        <termish> 
        [ ['||'|'|'] 
          [ <termish> || <.panic: "Null pattern not allowed"> ]
        ]*
        {*}
    }

    token termish {
        <.ws>
        <noun=quantified_atom>+
        {*}
    }

    token quantified_atom {
        <atom>
	<.ws>
        [ <quantifier> <.ws> ]?
        {*}
    }

    token atom {
        :dba('regex atom')
        [
        | \w [ \w+! <?before \w> ]?
        | <metachar>
        ]
        {*}
    }

    # proto token quantifier { <...> }
    token quantifier:sym<*> { $<sym>=['*'] <quantmod> {*} }
    token quantifier:sym<+> { $<sym>=['+'] <quantmod> {*} }
    token quantifier:sym<?> { $<sym>=['?'] <quantmod> {*} }

    token quantmod { ':'? [ '?' | '!' | '+' ]? {*} }

    # proto token metachar { <...> }
    token metachar:sym<[ ]> { '[' <nibbler> ']' {*} }
    token metachar:sym<.> { $<sym>=['.'] {*} }
    token metachar:sym<^> { $<sym>=['^'] {*} }
    token metachar:sym<^^> { $<sym>=['^^'] {*} }
    token metachar:sym<$> { $<sym>=['$'] {*} }
    token metachar:sym<$$> { $<sym>=['$$'] {*} }
    token metachar:sym<lwb> { $<sym>=['<<'|'«'] {*} }
    token metachar:sym<rwb> { $<sym>=['>>'|'»'] {*} }
    token metachar:sym<bs> { \\ <backslash> {*} }
    token metachar:sym<assert> { 
        '<' <assertion> 
        [ '>' || <.panic: "regex assertion not terminated by angle bracket"> ]
        {*}
    }

    # proto token backslash { <...> }
    token backslash:sym<w> { $<sym>=[<[dswnDSWN]>] {*} }
    token backslash:sym<b> { $<sym>=[<[bB]>] {*} }
    token backslash:sym<e> { $<sym>=[<[eE]>] {*} }
    token backslash:sym<f> { $<sym>=[<[fF]>] {*} }
    token backslash:sym<h> { $<sym>=[<[hH]>] {*} }
    token backslash:sym<r> { $<sym>=[<[rR]>] {*} }
    token backslash:sym<t> { $<sym>=[<[tT]>] {*} }
    token backslash:sym<v> { $<sym>=[<[vV]>] {*} }
    token backslash:sym<A> { 'A' <.obs: '\\A as beginning-of-string matcher;^'> }
    token backslash:sym<z> { 'z' <.obs: '\\z as end-of-string matcher;$'> }
    token backslash:sym<Z> { 'Z' <.obs: '\\Z as end-of-string matcher;\\n?$'> }
{'fraction_non_alphanumeric': 0.2842016094875053, 'fraction_numerical': 0.0006353240152477764, 'mean_word_length': 2.328400281888654, 'pattern_counts': {'":': 0, '<': 163, '<?xml version=': 0, '>': 163, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4305323', 'n_tokens_mistral': 2020, 'n_tokens_neox': 1929, 'n_words': 313}