<commit_before>import re
import random
from lxml import etree


def schema(relaxng):
    """
    Parse a RelaxNG schema document and return an etree instance of it
    :param relaxng: The RelaxNG schema as a string
    :type  relaxng: str

    :return: LXML etree RelaxNG instance
    :rtype : etree.RelaxNG
    """
    tree = etree.fromstring(relaxng)
    return etree.RelaxNG(tree)

def weighted_choice(choices):
    """
    Provides a weighted version of random.choice
    :param choices: A dictionary of choices, with the choice as the key and weight the value
    :type  choices: list of tuple of (str, int)
    """
    total = sum(weight for choice, weight in choices)
    rand = random.uniform(0, total)
    most = 0

    for choice, weight in choices:
        if most + weight > rand:
            return choice
        most += weight

def normalize(self, string):
    """
    Normalize input for comparison with other input
    :param string: The string to normalize
    :type  string: str

    :rtype: str
    """
    if not isinstance(string, str):
        self._log.warn('Attempted to normalize a non-string')
        return ''

    return re.sub(r'([^\s\w]|_)+', '', string.strip().casefold())<commit_msg>Normalize common bugfix<commit_after>import re
import random
from lxml import etree


def schema(relaxng):
    """
    Parse a RelaxNG schema document and return an etree instance of it
    :param relaxng: The RelaxNG schema as a string
    :type  relaxng: str

    :return: LXML etree RelaxNG instance
    :rtype : etree.RelaxNG
    """
    tree = etree.fromstring(relaxng)
    return etree.RelaxNG(tree)

def weighted_choice(choices):
    """
    Provides a weighted version of random.choice
    :param choices: A dictionary of choices, with the choice as the key and weight the value
    :type  choices: list of tuple of (str, int)
    """
    total = sum(weight for choice, weight in choices)
    rand = random.uniform(0, total)
    most = 0

    for choice, weight in choices:
        if most + weight > rand:
            return choice
        most += weight

def normalize(string):
    """
    Normalize input for comparison with other input
    :param string: The string to normalize
    :type  string: str

    :rtype: str
    """
    if not isinstance(string, str):
        return ''

    return re.sub(r'([^\s\w]|_)+', '', string.strip().casefold())
{'fraction_non_alphanumeric': 0.09427609427609428, 'fraction_numerical': 0.0016835016835016834, 'mean_word_length': 2.773015873015873, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1950943', 'n_tokens_mistral': 722, 'n_tokens_neox': 695, 'n_words': 265}