<commit_before>```
import ast
import textwrap
```

ignored Space = @/[ \t]+/
ignored Comment = @/#[^\r\n]*/

Newline = @/[\r\n][\s]*/
Sep = Some(Newline | ";")
Name = @/[_a-zA-Z][_a-zA-Z0-9]*/
Comma = wrap(",")

wrap(x) => Skip(Newline) >> x << Skip(Newline)

# Parse a full word, then see if it matches our keyword. The point is to make
# sure that we don't simply match the first part of a word. (For example, if
# the input string is "classify", we wouldn't want to match the keyword "class".)
kw(word) => Name where `lambda x: x == word`

Params = wrap("(") >> (wrap(Name) / Comma) << ")"

class StringLiteral {
    value: (
        @/(?s)("""([^\\]|\\.)*?""")/
        | @/(?s)('''([^\\]|\\.)*?''')/
        | @/("([^"\\]|\\.)*")/
        | @/('([^'\\]|\\.)*')/
    ) |> `ast.literal_eval`
}

class RegexLiteral {
    # Remove the leading "@/" and the trailing "/".
    value: @/\@\/([^\/\\]|\\.)*\// |> `lambda x: x[2:-1]`
}

class PythonSection {
    # Strip the backticks and remove any common indentation.
    value: @/(?s)```.*?```/ |> `lambda x: textwrap.dedent(x[3:-3])`
}

class PythonExpression {
    # Strip the backticks.
    value: @/`.*?`/ |> `lambda x: x[1:-1]`
}

class RuleDef {
    is_ignored: Opt(kw("ignored") | kw("ignore")) |> `bool`
    name: Name
    params: Opt(Params) << wrap("=>" | "=" | ":")
    expr: Expr
}

class ClassDef {
    name: kw("class") >> Name
    params: Opt(Params)
    fields: wrap("{") >> (RuleDef / Sep) << "}"
}

Stmt = ClassDef
    | RuleDef
    | PythonSection
    | PythonExpression

class LetExpression {
    name: kw("let") >> Name << wrap("=")
    expr: Expr << wrap(kw("in"))
    body: Expr
}

class Ref {
    value: Name
}

class ListLiteral {
    elements: "[" >> (wrap(Expr) / Comma) << "]"
}

Atom = ("(" >> wrap(Expr) << ")")
    | LetExpression
    | Ref
    | StringLiteral
    | RegexLiteral
    | ListLiteral
    | PythonExpression

class KeywordArg {
    name: Name << ("=" | ":")
    expr: Expr
}

class ArgList {
    args: "(" >> (wrap(KeywordArg | Expr) / Comma) << ")"
}

Expr = OperatorPrecedence(
    Atom,
    Postfix(ArgList),
    Postfix("?" | "*" | "+"),
    LeftAssoc(wrap("//" | "/")),
    LeftAssoc(wrap("<<" | ">>")),
    LeftAssoc(wrap("<|" | "|>" | "where")),
    LeftAssoc(wrap("|")),
)

# TODO: Implement `End`.
start = Skip(Newline) >> (Stmt / Sep) # << End<commit_msg>Update metasyntax definition.<commit_after>```
import ast
import textwrap
```

ignored Space = @/[ \t]+/
ignored Comment = @/#[^\r\n]*/

Newline = @/[\r\n][\s]*/
Sep = Some(Newline | ";")
Name = @/[_a-zA-Z][_a-zA-Z0-9]*/
Comma = wrap(",")

wrap(x) => Skip(Newline) >> x << Skip(Newline)

# Parse a full word, then see if it matches our keyword. The point is to make
# sure that we don't simply match the first part of a word. (For example, if
# the input string is "classify", we wouldn't want to match the keyword "class".)
kw(word) => Name where `lambda x: x == word`

Params = wrap("(") >> (wrap(Name) / Comma) << ")"

class StringLiteral {
    value: (
        @/(?s)("""([^\\]|\\.)*?""")/
        | @/(?s)('''([^\\]|\\.)*?''')/
        | @/("([^"\\]|\\.)*")/
        | @/('([^'\\]|\\.)*')/
    ) |> `ast.literal_eval`
}

class RegexLiteral {
    # Remove the leading "@/" and the trailing "/".
    value: @/\@\/([^\/\\]|\\.)*\// |> `lambda x: x[2:-1]`
}

class PythonSection {
    # Strip the backticks and remove any common indentation.
    value: @/(?s)```.*?```/ |> `lambda x: textwrap.dedent(x[3:-3])`
}

class PythonExpression {
    # Strip the backticks.
    value: @/`.*?`/ |> `lambda x: x[1:-1]`
}

class RuleDef {
    is_ignored: Opt(kw("ignored") | kw("ignore")) |> `bool`
    name: Name
    params: Opt(Params) << wrap("=>" | "=" | ":")
    expr: Expr
}

class ClassDef {
    name: kw("class") >> Name
    params: Opt(Params)
    fields: wrap("{") >> (RuleDef / Sep) << "}"
}

Stmt = ClassDef
    | RuleDef
    | PythonSection
    | PythonExpression

class LetExpression {
    name: kw("let") >> Name << wrap("=")
    expr: Expr << wrap(kw("in"))
    body: Expr
}

class Ref {
    value: Name
}

class ListLiteral {
    elements: "[" >> (wrap(Expr) / Comma) << "]"
}

Atom = ("(" >> wrap(Expr) << ")")
    | LetExpression
    | Ref
    | StringLiteral
    | RegexLiteral
    | ListLiteral
    | PythonExpression

class KeywordArg {
    name: Name << ("=" | ":")
    expr: Expr
}

class ArgList {
    args: "(" >> (wrap(KeywordArg | Expr) / Comma) << ")"
}

Expr = OperatorPrecedence(
    Atom,
    Postfix(ArgList),
    Postfix("?" | "*" | "+"),
    LeftAssoc(wrap("//" | "/")),
    LeftAssoc(wrap("<<" | ">>")),
    LeftAssoc(wrap("<|" | "|>" | "where")),
    LeftAssoc(wrap("|")),
)

start = Skip(Newline) >> (Stmt / Sep)
{'fraction_non_alphanumeric': 0.2497870528109029, 'fraction_numerical': 0.0034071550255536627, 'mean_word_length': 2.7396496815286624, 'pattern_counts': {'":': 4, '<': 51, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1821385', 'n_tokens_mistral': 1828, 'n_tokens_neox': 1797, 'n_words': 411}