<commit_before>from _ppeg import Pattern as P

EndOfFile = -P(1)
EndOfLine = P("\r\n") | P("\r") | P("\n")
Space = P(" ") | P("\t") | P.Var("EndOfLine")
Comment = P("#") + (-P.Var("EndOfLine") + P(1)) ** 0 + P.Var("EndOfLine")
Spacing = (P.Var("Space") | P.Var("Comment")) ** 0

LEFTARROW = P("<-") + P.Var("Spacing")
SLASH = P("/") + P.Var("Spacing")
AND = P("&") + P.Var("Spacing")
NOT = P("!") + P.Var("Spacing")
QUESTION = P("?") + P.Var("Spacing")
STAR = P("*") + P.Var("Spacing")
PLUS = P("+") + P.Var("Spacing")
OPEN = P("(") + P.Var("Spacing")
CLOSE = P(")") + P.Var("Spacing")
DOT = P(".") + P.Var("Spacing")

Char = ( P("\\") + P.Set("nrt'\"*[]\\") |
       P("\\") + P.Range("02") + P.Range("07") + P.Range("07") |
       P("\\") + P.Range("07") + P.Range("07") ** -1 |
       -P("\\") + P(1) )

Range = P.Var("Char") + P("-") + P.Var("Char") | P.Var("Char") + P.Var("Spacing")
Class = P("[") + ( -P("]") + P.Var("Range") ) ** 0 + P("]") + P.Var("Spacing")
Literal = (P("'") + ( -P("'") + P.Var("Char") ) ** 0 + P("'") + P.Var("Spacing") |
          P('"') + ( -P('"') + P.Var("Char") ) ** 0 + P('"') + P.Var("Spacing")
          )

Identifier = P.Var("IdentStart") + P.Var("IdentCont") ** 0 + P.Var("Spacing")
IdentStart = P.Range("azAZ") | P("_")
IdentCont = P.Var("IdentStart") | P.Range("09")

Primary = (P.Var("Identifier") + -P.Var("LEFTARROW") |
          P.Var("OPEN") + P.Var("Expression") + P.Var("CLOSE") |
          P.Var("Literal") |
          P.Var("Class") |
          P.Var("DOT")) + P.Var("Spacing")

Suffix = (P.Var("Primary") + 
          (P.Var("QUESTION") | P.Var("STAR") | P.Var("PLUS")) ** -1 +
          P.Var("Spacing"))
Prefix = (P.Var("AND") | P.Var("NOT")) ** -1 + P.Var("Suffix")
Sequence = P.Var("Prefix") ** 0
Expression = P.Var("Sequence") + (P.Var("SLASH") + P.Var("Sequence")) ** 0

Definition = P.Var("Identifier") + P.Var("LEFTARROW") + P.Var("Expression")

Grammar = P.Var("Spacing") + P.Var("Definition") ** 1 + P.Var("EndOfFile")

PEG = P.Grammar(
        Grammar,
        Grammar = Grammar,
        Definition = Definition,
        Expression = Expression,
        Sequence = Sequence,
        Prefix = Prefix,
        Suffix = Suffix,
        Primary = Primary,
        Identifier = Identifier,
        IdentStart = IdentStart,
        IdentCont = IdentCont,
        Literal = Literal,
        Class = Class,
        Range = Range,
        Char = Char,
        LEFTARROW = LEFTARROW,
        SLASH = SLASH,
        AND = AND,
        NOT = NOT,
        QUESTION = QUESTION,
        STAR = STAR,<commit_msg>Use less verbose way of referencing variables in pegmatcher<commit_after>from _ppeg import Pattern as P

class Var:
    def __getattr__(self, name):
        return P.Var(name)
V = Var()

EndOfFile = -P(1)
EndOfLine = P("\r\n") | P("\r") | P("\n")
Space = P(" ") | P("\t") | V.EndOfLine
Comment = P("#") + (-V.EndOfLine + P(1)) ** 0 + V.EndOfLine
Spacing = (V.Space | V.Comment) ** 0

LEFTARROW = P("<-") + V.Spacing
SLASH = P("/") + V.Spacing
AND = P("&") + V.Spacing
NOT = P("!") + V.Spacing
QUESTION = P("?") + V.Spacing
STAR = P("*") + V.Spacing
PLUS = P("+") + V.Spacing
OPEN = P("(") + V.Spacing
CLOSE = P(")") + V.Spacing
DOT = P(".") + V.Spacing

Char = ( P("\\") + P.Set("nrt'\"*[]\\") |
       P("\\") + P.Range("02") + P.Range("07") + P.Range("07") |
       P("\\") + P.Range("07") + P.Range("07") ** -1 |
       -P("\\") + P(1) )

Range = V.Char + P("-") + V.Char | V.Char + V.Spacing
Class = P("[") + ( -P("]") + V.Range ) ** 0 + P("]") + V.Spacing
Literal = (P("'") + ( -P("'") + V.Char ) ** 0 + P("'") + V.Spacing |
          P('"') + ( -P('"') + V.Char ) ** 0 + P('"') + V.Spacing
          )

Identifier = V.IdentStart + V.IdentCont ** 0 + V.Spacing
IdentStart = P.Range("azAZ") | P("_")
IdentCont = V.IdentStart | P.Range("09")

Primary = (V.Identifier + -V.LEFTARROW |
          V.OPEN + V.Expression + V.CLOSE |
          V.Literal |
          V.Class |
          V.DOT) + V.Spacing

Suffix = (V.Primary + 
          (V.QUESTION | V.STAR | V.PLUS) ** -1 +
          V.Spacing)
Prefix = (V.AND | V.NOT) ** -1 + V.Suffix
Sequence = V.Prefix ** 0
Expression = V.Sequence + (V.SLASH + V.Sequence) ** 0

Definition = V.Identifier + V.LEFTARROW + V.Expression

Grammar = V.Spacing + V.Definition ** 1 + V.EndOfFile

PEG = P.Grammar(
        start = "Grammar",
        Grammar = Grammar,
        Definition = Definition,
        Expression = Expression,
        Sequence = Sequence,
        Prefix = Prefix,
        Suffix = Suffix,
        Primary = Primary,
        Identifier = Identifier,
        IdentStart = IdentStart,
        IdentCont = IdentCont,
        Literal = Literal,
        Class = Class,
        Range = Range,
        Char = Char,
        LEFTARROW = LEFTARROW,
        SLASH = SLASH,
        AND = AND,
        NOT = NOT,
        QUESTION = QUESTION,
        STAR = STAR,
{'fraction_non_alphanumeric': 0.24310983134512545, 'fraction_numerical': 0.011106540518305225, 'mean_word_length': 2.374739764052741, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4841362', 'n_tokens_mistral': 1917, 'n_tokens_neox': 1865, 'n_words': 393}