<commit_before>#vim: set fileencoding=utf-8
import re

from regulations.generator import node_types


def appendix_supplement(data):
    """Handle items pointing to an appendix or supplement"""
    node_type = node_types.type_from_label(data['index'])
    if len(data['index']) == 2 and node_type in (node_types.APPENDIX,
                                                 node_types.INTERP):
        element = {}
        if node_type == node_types.INTERP:
            element['is_supplement'] = True
        else:
            element['is_appendix'] = True

        segments = try_split(data['title'])
        if segments:
            element['label'], element['sub_label'] = segments[:2]
        element['section_id'] = '-'.join(data['index'])
        return element


def try_split(text, chars=(u'—', '-')):
    """Utility method for splitting a string by one of multiple chars"""
    for c in chars:
        segments = text.split(c)
        if len(segments) > 1:
            return [s.strip() for s in segments]


def section(data):
    """ Parse out parts of a section title. """
    if len(data['index']) == 2 and data['index'][1].isdigit():
        element = {}
        element['is_section'] = True
        element['label'] = '.'.join(data['index'])
        element['section_id'] = '-'.join(data['index'])
        element['sub_label'] = re.search(
            element['label'] + r'[^\w\[]*(.*)', data['title']).group(1)
        return element
<commit_msg>Appendix titles now account for 'Reserved' and have a sane default<commit_after>#vim: set fileencoding=utf-8
import re

from regulations.generator import node_types


def appendix_supplement(data):
    """Handle items pointing to an appendix or supplement"""
    node_type = node_types.type_from_label(data['index'])
    if len(data['index']) == 2 and node_type in (node_types.APPENDIX,
                                                 node_types.INTERP):
        element = {}
        if node_type == node_types.INTERP:
            element['is_supplement'] = True
        else:
            element['is_appendix'] = True

        segments = try_split(data['title'])
        if segments:
            element['label'], element['sub_label'] = segments[:2]
        elif '[' in data['title']:
            position = data['title'].find('[')
            element['label'] = data['title'][:position]
            element['sub_label'] = data['title'][position:]
        else:
            element['label'] = data['title']

        element['section_id'] = '-'.join(data['index'])
        return element


def try_split(text, chars=(u'—', '-')):
    """Utility method for splitting a string by one of multiple chars"""
    for c in chars:
        segments = text.split(c)
        if len(segments) > 1:
            return [s.strip() for s in segments]


def section(data):
    """ Parse out parts of a section title. """
    if len(data['index']) == 2 and data['index'][1].isdigit():
        element = {}
        element['is_section'] = True
        element['label'] = '.'.join(data['index'])
        element['section_id'] = '-'.join(data['index'])
        element['sub_label'] = re.search(
            element['label'] + r'[^\w\[]*(.*)', data['title']).group(1)
        return element

{'fraction_non_alphanumeric': 0.15763394239702694, 'fraction_numerical': 0.004335707649427067, 'mean_word_length': 2.488120950323974, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6525447', 'n_tokens_mistral': 941, 'n_tokens_neox': 915, 'n_words': 234}