<commit_before># -*- coding: utf-8 -*-
#
# Pygments lexer for Cretonne.

from pygments.lexer import RegexLexer, bygroups
from pygments.token import *

class CretonneLexer(RegexLexer):
    name = 'Cretonne'
    aliases = ['cton']
    filenames = ['*.cton']

    tokens = {
        'root': [
            (r';.*?$', Comment.Single),
            (r'\b(function|entry)\b', Keyword),
            (r'\b(align)\b', Name.Attribute),
            # Well known value types.
            (r'\b(bool|i\d+|f32|f64)(x\d+)?\b', Keyword.Type),
            (r'\d+', Number.Integer),
            (r'0[xX][0-9a-fA-F]+', Number.Hex),
            # v<nn> = value
            # ss<nn> = stack slot
            (r'(v|ss)\d+', Name.Variable),
            # ebb<nn> = extended basic block
            (r'(ebb)\d+', Name.Label),
            (r'(=)( *)([a-z]\w*)', bygroups(Operator, Whitespace, Name.Function)),
            (r'^( +)([a-z]\w*\b)(?! *[,=])', bygroups(Whitespace, Name.Function)),
            (r'[a-z]\w*', Name),
            (r'->|=|:', Operator),
            (r'[{}(),.]', Punctuation),<commit_msg>Add hexadecimal numbers to the lexer.<commit_after># -*- coding: utf-8 -*-
#
# Pygments lexer for Cretonne.

from pygments.lexer import RegexLexer, bygroups, words
from pygments.token import *

def keywords(*args):
    return words(args, prefix=r'\b', suffix=r'\b')

class CretonneLexer(RegexLexer):
    name = 'Cretonne'
    aliases = ['cton']
    filenames = ['*.cton']

    tokens = {
        'root': [
            (r';.*?$', Comment.Single),
            # Strings are in double quotes, support \xx escapes only.
            (r'"([^"\\]+|\\[0-9a-fA-F]{2})*"', String),
            # A naked function name following 'function' is also a string.
            (r'\b(function)([ \t]+)(\w+)\b', bygroups(Keyword, Whitespace, String.Symbol)),
            # Numbers.
            (r'[-+]?0[xX][0-9a-fA-F]+', Number.Hex),
            (r'[-+]?0[xX][0-9a-fA-F]*\.[0-9a-fA-F]*([pP]\d+)?', Number.Hex),
            (r'[-+]?\d+\.\d+([eE]\d+)?', Number.Float),
            (r'[-+]?\d+', Number.Integer),
            # Reserved words.
            (keywords('function', 'entry'), Keyword),
            # Known attributes.
            (keywords('align'), Name.Attribute),
            # Well known value types.
            (r'\b(bool|i\d+|f32|f64)(x\d+)?\b', Keyword.Type),
            # v<nn> = value
            # ss<nn> = stack slot
            (r'(v|ss)\d+', Name.Variable),
            # ebb<nn> = extended basic block
            (r'(ebb)\d+', Name.Label),
            # Match instruction names in context.
            (r'(=)( *)([a-z]\w*)', bygroups(Operator, Whitespace, Name.Function)),
            (r'^( +)([a-z]\w*\b)(?! *[,=])', bygroups(Whitespace, Name.Function)),
            # Other names: results and arguments
            (r'[a-z]\w*', Name),
            (r'->|=|:', Operator),
            (r'[{}(),.]', Punctuation),
{'fraction_non_alphanumeric': 0.22700654495349637, 'fraction_numerical': 0.00826730967964175, 'mean_word_length': 2.015576323987539, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6915141', 'n_tokens_mistral': 1070, 'n_tokens_neox': 1032, 'n_words': 195}