<commit_before>from pygments.lexer import RegexLexer, bygroups, include
from pygments.token import (Text, Keyword, Number, Comment, Punctuation, Name,
    String, Literal)


__all__ = [
    "PyPyLogLexer",
]

class PyPyLogLexer(RegexLexer):
    """
    Lexer for PyPy log files.

    *New in Pygments 1.5.*
    """
    name = "PyPy Log"
    aliases = ["pypylog", "pypy"]
    filenames = ["*.pypylog"]
    mimetypes = ['application/x-pypylog']

    tokens = {
        "root": [
            (r"\[\w+\] {jit-log-.*?$", Keyword, "jit-log"),
            (r"\[\w+\] {jit-backend-counts$", Keyword, "jit-backend-counts"),
            include("extra-stuff"),
        ],
        "jit-log": [
            (r"\[\w+\] jit-log-.*?}$", Keyword, "#pop"),

            (r"[ifp]\d+", Name),
            (r"ptr\d+", Name),
            (r"(\()([\w_]+)(\))", bygroups(Punctuation, Name.Builtin, Punctuation)),
            (r"[\[\]=,()]", Punctuation),
            (r"(\d+\.\d+|inf|-inf)", Number.Float),
            (r"-?\d+", Number.Integer),
            (r"'.*'", String),
            (r"(None|descr|ConstClass|ConstPtr)", Name),
            (r"<.*?>", Name.Builtin),
            (r"(debug_merge_point|jump|finish)", Name.Class),
            (r"(int_add_ovf|int_add|int_sub_ovf|int_sub|int_mul_ovf|int_mul|int_or|"
             r"int_ge|int_gt|int_lt|int_is_zero|int_is_true|"
             r"uint_floordiv|"
             r"float_add|float_sub|float_mul|float_truediv|"
             r"float_eq|float_ne|float_gt|"
             r"ptr_eq|"
             r"force_token|"
             r"call_may_force|call_assembler|call|"
             r"new_with_vtable|new_array|"
             r"arraylen_gc|"
             r"getarrayitem_gc|setarrayitem_gc|getarrayitem_raw|setarrayitem_raw|"
             r"getfield_gc_pure|getfield_gc|getfield_raw|setfield_gc|setfield_raw|"
             r"guard_true|guard_false|guard_value|guard_isnull|guard_nonnull_class|guard_class|guard_no_overflow|guard_not_forced|guard_no_exception)", Name.Builtin),<commit_msg>Small update to the pypy lexer for missing words.<commit_after>from pygments.lexer import RegexLexer, bygroups, include
from pygments.token import (Text, Keyword, Number, Comment, Punctuation, Name,
    String, Literal)


__all__ = [
    "PyPyLogLexer",
]

class PyPyLogLexer(RegexLexer):
    """
    Lexer for PyPy log files.

    *New in Pygments 1.5.*
    """
    name = "PyPy Log"
    aliases = ["pypylog", "pypy"]
    filenames = ["*.pypylog"]
    mimetypes = ['application/x-pypylog']

    tokens = {
        "root": [
            (r"\[\w+\] {jit-log-.*?$", Keyword, "jit-log"),
            (r"\[\w+\] {jit-backend-counts$", Keyword, "jit-backend-counts"),
            include("extra-stuff"),
        ],
        "jit-log": [
            (r"\[\w+\] jit-log-.*?}$", Keyword, "#pop"),

            (r"[ifp]\d+", Name),
            (r"ptr\d+", Name),
            (r"(\()([\w_]+)(\))", bygroups(Punctuation, Name.Builtin, Punctuation)),
            (r"[\[\]=,()]", Punctuation),
            (r"(\d+\.\d+|inf|-inf)", Number.Float),
            (r"-?\d+", Number.Integer),
            (r"'.*'", String),
            (r"(None|descr|ConstClass|ConstPtr)", Name),
            (r"<.*?>", Name.Builtin),
            (r"(debug_merge_point|jump|finish)", Name.Class),
            (r"(int_add_ovf|int_add|int_sub_ovf|int_sub|int_mul_ovf|int_mul|int_or|"
             r"int_ge|int_gt|int_lt|int_is_zero|int_is_true|"
             r"uint_floordiv|"
             r"float_add|float_sub|float_mul|float_truediv|"
             r"float_eq|float_ne|float_gt|"
             r"ptr_eq|"
             r"force_token|"
             r"call_may_force|call_assembler|call|"
             r"new_with_vtable|new_array|"
             r"arraylen_gc|"
             r"getarrayitem_gc_pure|getarrayitem_gc|setarrayitem_gc|getarrayitem_raw|setarrayitem_raw|"
             r"getfield_gc_pure|getfield_gc|getfield_raw|setfield_gc|setfield_raw|"
             r"guard_true|guard_false|guard_value|guard_isnull|guard_nonnull_class|guard_class|guard_no_overflow|guard_not_forced|guard_no_exception)", Name.Builtin),
{'fraction_non_alphanumeric': 0.17501833292593497, 'fraction_numerical': 0.0009777560498655585, 'mean_word_length': 2.576923076923077, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '90545', 'n_tokens_mistral': 1562, 'n_tokens_neox': 1525, 'n_words': 191}