<commit_before>from pygments.lexer import RegexLexer, bygroups
from pygments.token import *

class JoedbiLexer(RegexLexer):
    name = 'joedbi'
    aliases = ['joedbi']
    filenames = ['*.joedbi']

    tokens = {
        'root': [
            (r'create_table\s+', Keyword, 'table'),
            (r'add_field\s+', Keyword, 'table_field_type'),
            (r'custom\s+', Keyword, 'custom'),
            (r'comment\s+', Keyword, 'literal'),
            (r'insert_into\s+', Keyword, 'table_literal'),
            (r'update\s+', Keyword, 'table_integer_field_literal'),
            (r'delete_from\s+', Keyword, 'table_integer'),
            (r'timestamp\s+', Keyword, 'timestamp'),
            (r'.+', Literal)
        ],
        'table': [
            (r'[a-zA-Z_]\w*', Name.Class)
        ],
        'table_field_type': [
            (r'[a-zA-Z_]\w*\s+', Name.Class, 'field_type')
        ],
        'field_type': [
            (r'[a-zA-Z_]\w*\s+', Name.Variable, 'type')
        ],
        'type': [
            (r'references\s+[a-zA-Z_]\w*', Keyword.Type),
            (r'[a-zA-Z_]\w*', Keyword.Type)
        ],
        'custom': [
            (r'[a-zA-Z_]\w*', Name.Function)
        ],
        'literal': [
            (r"\"([^\\\"]|\\.)*\"", Literal.String),
            (r"\d+", Number.Integer)<commit_msg>pygment syntax works with multi_server.joedbi<commit_after>from pygments.lexer import RegexLexer, bygroups
from pygments.token import *

class JoedbiLexer(RegexLexer):
    name = 'joedbi'
    aliases = ['joedbi']
    filenames = ['*.joedbi']

    tokens = {
        'root': [
            (r'create_table\s+', Keyword, 'table'),
            (r'add_field\s+', Keyword, 'table_field_type'),
            (r'custom\s+', Keyword, 'custom'),
            (r'comment\s+', Keyword, 'literal'),
            (r'insert_into\s+', Keyword, 'table_literal'),
            (r'update\s+', Keyword, 'table_integer_field_literal'),
            (r'delete_from\s+', Keyword, 'table_integer'),
            (r'timestamp\s+', Keyword, 'timestamp'),
            (r'.+', Literal)
        ],
        'table': [
            (r'[a-zA-Z_]\w*', Name.Class)
        ],
        'table_field_type': [
            (r'[a-zA-Z_]\w*\s+', Name.Class, 'field_type')
        ],
        'field_type': [
            (r'[a-zA-Z_]\w*\s+', Name.Variable, 'type')
        ],
        'type': [
            (r'references\s+[a-zA-Z_]\w*', Keyword.Type),
            (r'[a-zA-Z_]\w*', Keyword.Type)
        ],
        'custom': [
            (r'[a-zA-Z_]\w*', Name.Function)
        ],
        'literal': [
            (r'\s*\"([^\\\"]|\\.)*\"', Literal.String),
            (r'\s*\d+', Number.Integer)
{'fraction_non_alphanumeric': 0.212406015037594, 'fraction_numerical': 0.0, 'mean_word_length': 1.8551502145922747, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3373882', 'n_tokens_mistral': 997, 'n_tokens_neox': 970, 'n_words': 139}