<commit_before># -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# Copyright 2015-2016 by EcpyPulses Authors, see AUTHORS for more details.
#
# Distributed under the terms of the BSD license.
#
# The full license is in the file LICENCE, distributed with this software.
# -----------------------------------------------------------------------------
"""

"""
from __future__ import (division, unicode_literals, print_function,
                        absolute_import)

# XXXX should use the transformers from ecpy
# TODO would be nicer with regex but I never managed to get the regex right.
def normalize_sequence_name(name):
    """Normalize names.

    For tasks, replaces '_' by spaces and add spaces between 'aA' sequences.
    For templates, only the extension file is removed.

    """
    if name.endswith('.temp_pulse.ini'):
        name, _, _ = name.rsplit('.', 2)
        return name.capitalize()

    if name.endswith('Sequence'):
        name = name[:-8] + '\0'

    package = None
    if '.' in name:
        package, name = name.rsplit('.', 1)

    aux = ''
    n_len = len(name)
    for i, char in enumerate(name):
        if char == '_':
            aux += ' '
            continue

        if char != '\0':
            if char.isupper() and i != 0:
                if name[i-1].islower():
                    if i+1 != n_len and name[i+1].islower():
                        aux += ' ' + char.lower()
                    else:
                        aux += ' ' + char
                else:
                    if i+1 != n_len and name[i+1].islower():
                        aux += ' ' + char.lower()
                    else:
                        aux += char
            else:
                if i == 0:
                    aux += char.upper()
                else:
                    aux += char

    return package + '.' + aux if package else aux


# TODO would be nicer with regex but I never managed to get the regex right.
def normalize_context_name(name):
    """Normalize names.

    For tasks, replaces '_' by spaces and add spaces between 'aA' sequences.
    For templates, only the extension file is removed.

    """
    if name.endswith('Context'):
        name = name[:-7] + '\0'

    package = None
    if '.' in name:
        package, name = name.rsplit('.', 1)

    aux = ''
    for i, char in enumerate(name):
        if char == '_':
            aux += ' '
            continue

        if char != '\0':
            if char.isupper() and i != 0:
                if name[i-1].islower():
                    if name[i+1].islower():
                        aux += ' ' + char.lower()
                    else:
                        aux += ' ' + char
                elif i + 1 < len(name):
                    if name[i+1].islower():
                        aux += ' ' + char.lower()
                    else:
                        aux += char
            else:
                if i == 0:
                    aux += char.upper()
                else:
                    aux += char

    return package + '.' + aux if package else aux<commit_msg>pulses : add a normalizer for shape names<commit_after># -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# Copyright 2015-2016 by EcpyPulses Authors, see AUTHORS for more details.
#
# Distributed under the terms of the BSD license.
#
# The full license is in the file LICENCE, distributed with this software.
# -----------------------------------------------------------------------------
"""Utility functions to build user-friendly names from ids.

Those should be use as formatting function argument to the ids_to_unique_names
function found in ecpy.utils.transformers

"""
from __future__ import (division, unicode_literals, print_function,
                        absolute_import)


# TODO would be nicer with regex but I never managed to get the regex right.
def normalize_sequence_name(name):
    """Normalize sequence names.

    For sequences, strip the terminal Sequence if present, replaces '_' by
    spaces and add spaces between 'aA' sequences.
    For templates, only the extension file is removed.

    """
    if name.endswith('.temp_pulse.ini'):
        name, _, _ = name.rsplit('.', 2)
        return name.capitalize()

    if name.endswith('Sequence'):
        name = name[:-8] + '\0'

    package = None
    if '.' in name:
        package, name = name.rsplit('.', 1)

    aux = ''
    n_len = len(name)
    for i, char in enumerate(name):
        if char == '_':
            aux += ' '
            continue

        if char != '\0':
            if char.isupper() and i != 0:
                if name[i-1].islower():
                    if i+1 != n_len and name[i+1].islower():
                        aux += ' ' + char.lower()
                    else:
                        aux += ' ' + char
                else:
                    if i+1 != n_len and name[i+1].islower():
                        aux += ' ' + char.lower()
                    else:
                        aux += char
            else:
                if i == 0:
                    aux += char.upper()
                else:
                    aux += char

    return package + '.' + aux if package else aux


# TODO would be nicer with regex but I never managed to get the regex right.
def normalize_context_name(name):
    """Normalize context names.

    Strip terminal Context and replaces '_' by spaces and add spaces between
    'aA' sequences.

    """
    if name.endswith('Context'):
        name = name[:-7] + '\0'

    package = None
    if '.' in name:
        package, name = name.rsplit('.', 1)

    aux = ''
    for i, char in enumerate(name):
        if char == '_':
            aux += ' '
            continue

        if char != '\0':
            if char.isupper() and i != 0:
                if name[i-1].islower():
                    if name[i+1].islower():
                        aux += ' ' + char.lower()
                    else:
                        aux += ' ' + char
                elif i + 1 < len(name):
                    if name[i+1].islower():
                        aux += ' ' + char.lower()
                    else:
                        aux += char
            else:
                if i == 0:
                    aux += char.upper()
                else:
                    aux += char

    return package + '.' + aux if package else aux


def normalize_shape_name(name):
    """Normalize shape names.

    Strip terminal Shape and replaces '_' by spaces and add spaces between
    'aA' sequences.

    """
    if name.endswith('Shape'):
        name = name[:-7] + '\0'

    package = None
    if '.' in name:
        package, name = name.rsplit('.', 1)

    aux = ''
    for i, char in enumerate(name):
        if char == '_':
            aux += ' '
            continue

        if char != '\0':
            if char.isupper() and i != 0:
                if name[i-1].islower():
                    if name[i+1].islower():
                        aux += ' ' + char.lower()
                    else:
                        aux += ' ' + char
                elif i + 1 < len(name):
                    if name[i+1].islower():
                        aux += ' ' + char.lower()
                    else:
                        aux += char
            else:
                if i == 0:
                    aux += char.upper()
                else:
                    aux += char

    return package + '.' + aux if package else aux
{'fraction_non_alphanumeric': 0.15293028178330467, 'fraction_numerical': 0.009525069453631433, 'mean_word_length': 1.1862348178137652, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3418663', 'n_tokens_mistral': 2070, 'n_tokens_neox': 1886, 'n_words': 651}