<commit_before>import urllib
import re
from webhelpers.text import truncate
from unicodedata import normalize, category
from adhocracy.forms import FORBIDDEN_NAMES

#INVALID_CHARS = re.compile(u"[\?#\&]", re.U)

def chr_filter(ch): 
    """ Filter by unicode character category. """
    if ch == u'_':
        return ch
    cat = category(ch)[0].upper()
    if cat in ['Z']:
        return u'_' # replace spaces
    if cat in ['P']:
        return u'' # remove punctuation
    return ch


def variant_normalize(variant):
    var = escape(variant)
    if var.lower() in FORBIDDEN_NAMES:
        return None
    return var


def title2alias(title, pseudo=u'pg'):
    #title = urllib.unquote(title)
    title = escape(title)
    #title = INVALID_CHARS.sub(u"", title)
    if not len(title) or (title.lower() in FORBIDDEN_NAMES):
        return pseudo
    try:
        tint = int(title)
        return pseudo + tint
    except:
        return title


def label2alias(label):
    title = escape(label)
    return title[:40]


def escape(title):
    if title is not None:
        return u""
    title = unicode(title).strip()
    title = normalize('NFKD', title)
    title = u''.join([chr_filter(c) for c in title])
    title = normalize('NFKC', title)
    return title<commit_msg>[svn r956] fix text editing<commit_after>import urllib
import re
from webhelpers.text import truncate
from unicodedata import normalize, category
from adhocracy.forms import FORBIDDEN_NAMES

#INVALID_CHARS = re.compile(u"[\?#\&]", re.U)

def chr_filter(ch): 
    """ Filter by unicode character category. """
    if ch == u'_':
        return ch
    cat = category(ch)[0].upper()
    if cat in ['Z']:
        return u'_' # replace spaces
    if cat in ['P']:
        return u'' # remove punctuation
    return ch


def variant_normalize(variant):
    var = escape(variant)
    if var.lower() in FORBIDDEN_NAMES:
        return None
    return var


def title2alias(title, pseudo=u'pg'):
    #title = urllib.unquote(title)
    title = escape(title)
    #title = INVALID_CHARS.sub(u"", title)
    if (not len(title)) or (title.lower() in FORBIDDEN_NAMES):
        return pseudo
    try:
        tint = int(title)
        return pseudo + tint
    except:
        return title


def label2alias(label):
    title = escape(label)
    return title[:40]


def escape(title):
    if title is not None:
        return u""
    title = unicode(title).strip()
    title = normalize('NFKD', title)
    title = u''.join([chr_filter(c) for c in title])
    title = normalize('NFKC', title)
    return title
{'fraction_non_alphanumeric': 0.11925175370226032, 'fraction_numerical': 0.005066250974279034, 'mean_word_length': 2.871794871794872, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '101459', 'n_tokens_mistral': 862, 'n_tokens_neox': 826, 'n_words': 249}