<commit_before>import keyword
import _jsre as re

from browser import html

letters = 'abcdefghijklmnopqrstuvwxyz'
letters += letters.upper()+'_'
digits = '0123456789'

builtin_funcs = """abs|dict|help|min|setattr|
all|dir|hex|next|slice|
any|divmod|id|object|sorted|
ascii|enumerate|input|oct|staticmethod|
bin|eval|int|open|str|
bool|exec|isinstance|ord|sum|
bytearray|filter|issubclass|pow|super|
bytes|float|iter|print|tuple|
callable|format|len|property|type|
chr|frozenset|list|range|vars|
classmethod|getattr|locals|repr|zip|
compile|globals|map|reversed|__import__|
complex|hasattr|max|round|
delattr|hash|memoryview|set|
"""

kw_pattern = '^('+'|'.join(keyword.kwlist)+')$'
bf_pattern = '^('+builtin_funcs.replace("\n", "")+')$'

def escape(txt):
    txt = txt.replace('<', '&lt;')
    txt = txt.replace('>', '&gt;')
    return txt

def highlight(txt):
    res = html.PRE()
    i = 0
    name = ''
    while i < len(txt):
        car = txt[i]
        if car in ["'", '"']:
            found_match = False
            k = i + 1
            while k < len(txt):
                if txt[k] == car:
                    nb_as = 0
                    j = k - 1
                    while True:<commit_msg>Minor change in highlight.py<commit_after>import keyword
import _jsre as re

from browser import html

letters = 'abcdefghijklmnopqrstuvwxyz'
letters += letters.upper()+'_'
digits = '0123456789'

builtin_funcs = """abs|dict|help|min|setattr|
all|dir|hex|next|slice|
any|divmod|id|object|sorted|
ascii|enumerate|input|oct|staticmethod|
bin|eval|int|open|str|
bool|exec|isinstance|ord|sum|
bytearray|filter|issubclass|pow|super|
bytes|float|iter|print|tuple|
callable|format|len|property|type|
chr|frozenset|list|range|vars|
classmethod|getattr|locals|repr|zip|
compile|globals|map|reversed|__import__|
complex|hasattr|max|round|
delattr|hash|memoryview|set|
"""

kw_pattern = '^(' + '|'.join(keyword.kwlist) + ')$'
bf_pattern = '^(' + builtin_funcs.replace("\n", "") + ')$'

def escape(txt):
    txt = txt.replace('<', '&lt;')
    txt = txt.replace('>', '&gt;')
    return txt

def highlight(txt):
    res = html.PRE()
    i = 0
    name = ''
    while i < len(txt):
        car = txt[i]
        if car in ["'", '"']:
            found_match = False
            k = i + 1
            while k < len(txt):
                if txt[k] == car:
                    nb_as = 0
                    j = k - 1
                    while True:
{'fraction_non_alphanumeric': 0.16563912864775995, 'fraction_numerical': 0.011508425811755036, 'mean_word_length': 3.338680926916221, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '798062', 'n_tokens_mistral': 914, 'n_tokens_neox': 875, 'n_words': 158}