<commit_before>"""
    pygments.lexers.jsonnet
    ~~~~~~~~~~~~~~~~~~~~~~~

    Lexer for Jsonnet data templating language.

    :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""
from pygments.lexer import (
    include,
    RegexLexer,
    words,
)
from pygments.token import (
    Comment,
    Keyword,
    Name,
    Number,
    Operator,
    Punctuation,
    String,
    Whitespace,
)

__all__ = ['JsonnetLexer']

jsonnet_token_chars = r'[_A-Za-z0-9]'
jsonnet_token = r'[_A-Za-z]' + jsonnet_token_chars + '*'
jsonnet_function_token = jsonnet_token + r'(?=\()'


def string_rules(quote_mark):
    return [
        (r"[^{}\\]".format(quote_mark), String),
        (r"\\.", String.Escape),
        (quote_mark, String, '#pop'),
    ]


def quoted_field_name(quote_mark):
    return [
        (r'([^{quote}\\]|\\.)*{quote}'.format(quote=quote_mark),
         Name.Variable, 'field_separator')
    ]


class JsonnetLexer(RegexLexer):
    name = 'Jsonnet'<commit_msg>Switch to \w<commit_after>"""
    pygments.lexers.jsonnet
    ~~~~~~~~~~~~~~~~~~~~~~~

    Lexer for Jsonnet data templating language.

    :copyright: Copyright 2006-2022 by the Pygments team, see AUTHORS.
    :license: BSD, see LICENSE for details.
"""
from pygments.lexer import (
    include,
    RegexLexer,
    words,
)
from pygments.token import (
    Comment,
    Keyword,
    Name,
    Number,
    Operator,
    Punctuation,
    String,
    Whitespace,
)

__all__ = ['JsonnetLexer']

jsonnet_token = r'[^\W\d]\w*'
jsonnet_function_token = jsonnet_token + r'(?=\()'


def string_rules(quote_mark):
    return [
        (r"[^{}\\]".format(quote_mark), String),
        (r"\\.", String.Escape),
        (quote_mark, String, '#pop'),
    ]


def quoted_field_name(quote_mark):
    return [
        (r'([^{quote}\\]|\\.)*{quote}'.format(quote=quote_mark),
         Name.Variable, 'field_separator')
    ]


class JsonnetLexer(RegexLexer):
    name = 'Jsonnet'
{'fraction_non_alphanumeric': 0.17118473895582328, 'fraction_numerical': 0.009036144578313253, 'mean_word_length': 3.0100603621730384, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4595407', 'n_tokens_mistral': 739, 'n_tokens_neox': 685, 'n_words': 138}