<commit_before>import importlib

from pygments.lexer import RegexLexer, bygroups
from pygments.style import Style
from pygments.styles import STYLE_MAP
from pygments.token import *

def load_style(full_class_string):
    modulename, styleclass = full_class_string.split('::')
    module = importlib.import_module("pygments.styles." + modulename)
    return getattr(module, styleclass)

# probably don't need to preload all styles...
repl_styles = {}
for name, import_info in STYLE_MAP.items():
    repl_styles[name] = load_style(import_info)


class FranzLexer(RegexLexer):
    name = 'Franz Lexer'
    tokens = {
        'root': [
            (r'"', String.Double, 'double-quote'),
            (r'[0-9]+(\.[0-9]+)?', Number),
            (r'\b(if|else|for|while|in|to|fn|ⲗ|try|rescue|assert|include|yield|return|break|continue)\b', Keyword.Reserved),<commit_msg>fix ws issues in some styles (e.g. Tango)<commit_after>import importlib

from pygments.lexer import RegexLexer, bygroups
from pygments.styles import STYLE_MAP
from pygments.token import *


def load_style(full_class_string):
    modulename, styleclass = full_class_string.split('::')
    module = importlib.import_module("pygments.styles." + modulename)
    return getattr(module, styleclass)


repl_styles = {}
for name, import_info in STYLE_MAP.items():
    repl_styles[name] = load_style(import_info)
    repl_styles[name].styles[Whitespace] = '' # some styles underline ws


class FranzLexer(RegexLexer):
    name = 'Franz Lexer'
    tokens = {
        'root': [
            (r'"', String.Double, 'double-quote'),
            (r'[0-9]+(\.[0-9]+)?', Number),
            (r'\b(if|else|for|while|in|to|fn|ⲗ|try|rescue|assert|include|yield|return|break|continue)\b', Keyword.Reserved),
{'fraction_non_alphanumeric': 0.14122355631789593, 'fraction_numerical': 0.004574042309891366, 'mean_word_length': 4.223880597014926, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1596744', 'n_tokens_mistral': 618, 'n_tokens_neox': 589, 'n_words': 116}