<commit_before>import re
import unicodedata


HALF = ' '
CAP = '\ue302'


def space_priority(char):
    return {'L': 7, 'M': 7, 'N': 5, 'S': 3, 'P': 1,
            'Z': -1, 'C': -3}[unicodedata.category(char)[0]]


def tokenize(s, decap=False):
    """Simple reversible tokenizer"""

    toks = ['']
    current_cat = 0
    for c in s:
        cat = space_priority(c)
        if c == ' ':
            toks[-1] += HALF
            toks.append(HALF)
            current_cat = None
        elif current_cat is None:
            toks[-1] += c
            current_cat = cat
        elif cat == current_cat:
            toks[-1] += c # HALF + c
        else:
            if cat <= 0 and current_cat <= 0:
                toks.append(c)
            elif cat < current_cat:
                toks[-1] += HALF
                toks.append(c)
            else:
                toks.append(HALF + c)
            current_cat = cat
    if toks[0] == '':
        toks = toks[1:]<commit_msg>always separate punctuation as own token (#3)<commit_after>import re
import unicodedata


HALF = ' '
CAP = '\ue302'


def space_priority(char):
    return {'L': 7, 'M': 7, 'N': 5, 'S': 3, 'P': 1,
            'Z': -1, 'C': -3}[unicodedata.category(char)[0]]


def tokenize(s, decap=False, split_punctuation=True):
    """Simple reversible tokenizer"""

    toks = ['']
    current_cat = 0
    for c in s:
        cat = space_priority(c)
        if c == ' ':
            toks[-1] += HALF
            toks.append(HALF)
            current_cat = None
            continue
        elif current_cat is None:
            toks[-1] += c
        elif cat == current_cat and (cat > 2 or not split_punctuation):
            toks[-1] += c # HALF + c
        elif cat <= 0 and current_cat <= 0:
            toks.append(c)
        elif cat <= current_cat:
            toks[-1] += HALF
            toks.append(c)
        else:
            toks.append(HALF + c)
        current_cat = cat
    if toks[0] == '':
        toks = toks[1:]
{'fraction_non_alphanumeric': 0.14328808446455504, 'fraction_numerical': 0.021116138763197588, 'mean_word_length': 1.5911458333333333, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '725164', 'n_tokens_mistral': 707, 'n_tokens_neox': 668, 'n_words': 178}