<commit_before>from pygments.lexer import RegexLexer, bygroups, include, combined, words
from pygments.token import *
import sphinx

class JamLexer(RegexLexer):
    name = "Jam"
    aliases = ["jam"]
    filenames = ["*.jm"]

    INTEGER_REGEX = "[0-9]([0-9_]*[0-9])?"

    tokens = {
        'root': [
            ("#.*?$", Comment),
            include('constants'),
            include('keywords'),
            include('builtins'),
            (INTEGER_REGEX, Literal.Number),
            ("{0}\.({0})?".format(INTEGER_REGEX), Literal.Number),
            ("({0})?\.{0}".format(INTEGER_REGEX), Literal.Number),
            ("\"(.*)?\"", Literal.String),
            ("\(", Text, '#push'),
            ("\)", Text, '#pop'),
            (" ", Text.Whitespace),
            include('operators'),
            ("([a-zA-Z_][a-zA-Z_0-9]*)", Name),
        ],

        'keywords': [
            (words(
                (
                    "self",
                    "const",
                    "end",
                    "def",
                    "class",
                    "template",
                    "if",
                    "elif",
                    "else",
                    "while",
                    "for",
                    "in",
                    "as",
                    "import",
                ), suffix = r'\b'),
             Keyword)
        ],

        'constants': [
            (words(
                (
                    "true",
                    "false",
                    "null",
                    "inf",
                    "nan",
                ), suffix = r'\b'),
             Name.Builtin)
        ],

        'builtins': [
            (words(
                (
                    "puts",
                    "Int",
                    "Int8",
                    "Int16",
                    "Int32",
                    "Int64",
                    "Int128",
                    "UInt",
                    "UInt8",
                    "UInt16",
                    "UInt32",
                    "UInt64",
                    "UInt128",
                    "Float",
                    "Float16",
                    "Float32",
                    "Float64",
                    "UFloat",
                    "UFloat16",
                    "UFloat32",
                    "UFloat64",
                    "Bool",
                ), suffix = r'\b'),
             Name.Builtin)
        ],

        'operators': [
            (words(
                (
                    "~",
                    "~",
                    "!",
                    "%",
                    "\^",
                    "&",
                    "&&",
                    "*",
                    "**",
                    "-",
                    "-=",
                    "+",
                    "+=",
                    "=",
                    "==",
                    "!=",
                    "|",
                    "||",
                    ":",
                    "?",
                    "<",
                    "<=",
                    ">",
                    ">=",
                    ".",
                    "/",
                    "//",
                ), suffix = r'\b'),
             Name.Builtin)
        ],
    }<commit_msg>Fixed pygments lexer for operators<commit_after>from pygments.lexer import RegexLexer, bygroups, include, combined, words
from pygments.token import *
import sphinx

class JamLexer(RegexLexer):
    name = "Jam"
    aliases = ["jam"]
    filenames = ["*.jm"]

    INTEGER_REGEX = r"[0-9]([0-9_]*[0-9])?"

    tokens = {
        'root': [
            (r"#.*?$", Comment),
            include('constants'),
            include('keywords'),
            include('builtins'),
            (INTEGER_REGEX, Literal.Number),
            (r"{0}\.({0})?".format(INTEGER_REGEX), Literal.Number),
            (r"({0})?\.{0}".format(INTEGER_REGEX), Literal.Number),
            (r"\"(.*)?\"", Literal.String),
            (r"\(", Text, '#push'),
            (r"\)", Text, '#pop'),
            (" ", Text.Whitespace),
            include('operators'),
            (r"([a-zA-Z_][a-zA-Z_0-9]*)", Name),
        ],

        'keywords': [
            (words(
                (
                    "self",
                    "const",
                    "end",
                    "def",
                    "class",
                    "template",
                    "if",
                    "elif",
                    "else",
                    "while",
                    "for",
                    "in",
                    "as",
                    "import",
                ), suffix = r'\b'),
             Keyword)
        ],

        'constants': [
            (words(
                (
                    "true",
                    "false",
                    "null",
                    "inf",
                    "nan",
                ), suffix = r'\b'),
             Name.Builtin)
        ],

        'builtins': [
            (words(
                (
                    "puts",
                    "Int",
                    "Int8",
                    "Int16",
                    "Int32",
                    "Int64",
                    "Int128",
                    "UInt",
                    "UInt8",
                    "UInt16",
                    "UInt32",
                    "UInt64",
                    "UInt128",
                    "Float",
                    "Float16",
                    "Float32",
                    "Float64",
                    "UFloat",
                    "UFloat16",
                    "UFloat32",
                    "UFloat64",
                    "Bool",
                ), suffix = r'\b'),
             Name.Builtin)
        ],

        'operators': [(symbol, Operator) for symbol in (
                r"~",
                r"!",
                r"%",
                r"\^",
                r"&",
                r"&&",
                r"\*",
                r"\*\*",
                r"-",
                r"-=",
                r"\+",
                r"\+=",
                r"=",
                r"==",
                r"!=",
                r"\|",
                r"\|\|",
                r":",
                r"\?",
                r"<",
                r"<=",
                r">",
                r">=",
                r"\.",
                r"/",
                r"//",
            )
        ]
    }
{'fraction_non_alphanumeric': 0.15755976404843217, 'fraction_numerical': 0.013660353927351754, 'mean_word_length': 0.6229219143576826, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5483006', 'n_tokens_mistral': 1809, 'n_tokens_neox': 1583, 'n_words': 245}