<commit_before>import re, string
from difflib import SequenceMatcher, _calculate_ratio

from flatten import flatten

def multi_split(text, regexes):
    """
    Split the text by the given regexes, in priority order.

    Make sure that the regex is parenthesized so that matches are returned in
    re.split().

    Splitting on a single regex works like normal split.
    >>> '|'.join(multi_split('one two three', [r'(\w+)']))
    'one| |two| |three'

    Splitting on digits first separates the digits from their word
    >>> '|'.join(multi_split('one234five 678', [r'(\d+)', r'(\w+)']))
    'one|234|five| |678'

    Splitting on words first keeps the word with digits intact.
    >>> '|'.join(multi_split('one234five 678', [r'(\w+)', r'(\d+)']))
    'one234five| |678'
    """
    if not isinstance(regexes, (list, tuple)):
        raise ValueError('regexes argument should be a list, not %s' % type(regexes).__name__)

    def make_regex(s):
        return re.compile(s) if isinstance(s, basestring) else s
    regexes = [make_regex(r) for r in regexes]

    # Run the list of pieces through the regex split, splitting it into more
    # pieces. Once a piece has been matched, add it to finished_pieces and
    # don't split it again. The pieces should always join back together to form
    # the original text.
    def apply(regex, piece_list):
        for piece in piece_list:
            if piece in finished_pieces:
                yield piece
                continue
            for s in regex.split(piece):
                if regex.match(s):
                    finished_pieces.add(s)
                if s:
                    yield s

    piece_list = [text]
    finished_pieces = set()
    for regex in regexes:
        piece_list = list(apply(regex, piece_list))<commit_msg>Test coverage and refactoring in text.multi_split<commit_after>import re, string
from difflib import SequenceMatcher, _calculate_ratio

from flatten import flatten

def multi_split(text, regexes):
    """
    Split the text by the given regexes, in priority order.

    Make sure that the regex is parenthesized so that matches are returned in
    re.split().

    Splitting on a single regex works like normal split.
    >>> '|'.join(multi_split('one two three', [r'(\w+)']))
    'one| |two| |three'

    Splitting on digits first separates the digits from their word
    >>> '|'.join(multi_split('one234five 678', [r'(\d+)', r'(\w+)']))
    'one|234|five| |678'

    Splitting on words first keeps the word with digits intact.
    >>> '|'.join(multi_split('one234five 678', [r'(\w+)', r'(\d+)']))
    'one234five| |678'

    Regexes need to be parenthesized so that all re.split keeps all pieces.
    >>> '|'.join(multi_split('one two three', [r' ']))
    Traceback (most recent call last):
        ...
    ValueError: The regular expression ' ' did not preserve all pieces when splitting. It may not be parenthesized.
    """
    def make_regex(s):
        return re.compile(s) if isinstance(s, basestring) else s
    regexes = [make_regex(r) for r in regexes]

    # Run the list of pieces through the regex split, splitting it into more
    # pieces. Once a piece has been matched, add it to finished_pieces and
    # don't split it again. The pieces should always join back together to form
    # the original text.
    def apply(regex, piece_list):
        for piece in piece_list:
            if piece in finished_pieces:
                yield piece
                continue
            for s in regex.split(piece):
                if regex.match(s):
                    finished_pieces.add(s)
                if s:
                    yield s

    piece_list = [text]
    finished_pieces = set()
    for regex in regexes:
        piece_list = list(apply(regex, piece_list))
{'fraction_non_alphanumeric': 0.1052492046659597, 'fraction_numerical': 0.012725344644750796, 'mean_word_length': 2.607074569789675, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '877943', 'n_tokens_mistral': 1168, 'n_tokens_neox': 1066, 'n_words': 414}