<commit_before># -*- coding: utf-8 -*-
#
# Copyright 2014-2015 SpendRight, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Normalization of data, mostly strings."""
import re
import unicodedata

from titlecase import titlecase
from unidecode import unidecode

# matches all whitespace, including non-ascii (e.g. non-breaking space)
WHITESPACE_RE = re.compile(r'\s+', re.U)


BAD_CODEPOINTS = {
    # smart quotes
    0x2018: "'",
    0x2019: "'",
    0x201c: '"',
    0x201d: '"',
    # ligatures
    # from https://en.wikipedia.org/wiki/Typographic_ligature#Ligatures_in_Unicode_.28Latin_alphabets.29  # noqa
    0xfb00: 'ff',
    0xfb01: 'fi',
    0xfb02: 'fl',
    0xfb03: 'ffi',
    0xfb04: 'ffl',
    0xfb06: 'st',
}


def clean_string(s):
    """Clean messy strings from the outside world."""
    if not isinstance(s, str):
        raise TypeError

    s = unicodedata.normalize('NFKD', s)
    s = s.translate(BAD_CODEPOINTS)
    s = simplify_whitespace(s)

    return s


def simplify_whitespace(s):
    """Strip s, and use only single spaces within s."""
    return WHITESPACE_RE.sub(' ', s.strip())


def to_title_case(s):
    """Like titlecase.titlecase(), but treat hyphens as spaces."""
    return ''.join(
        '-' if s[i] == '-' else c
        for i, c in enumerate(titlecase(s.replace('-', ' '))))


def norm(s):
    """Remove accents and convert to lowercase."""
    return unidecode(s).lower()


def smunch(s):
    """Like norm(), except we remove whitespace too."""
    return WHITESPACE_RE.sub('', s)<commit_msg>whoops, norm() should include smunch()<commit_after># -*- coding: utf-8 -*-
#
# Copyright 2014-2015 SpendRight, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Normalization of data, mostly strings."""
import re
import unicodedata

from titlecase import titlecase
from unidecode import unidecode

# matches all whitespace, including non-ascii (e.g. non-breaking space)
WHITESPACE_RE = re.compile(r'\s+', re.U)


BAD_CODEPOINTS = {
    # smart quotes
    0x2018: "'",
    0x2019: "'",
    0x201c: '"',
    0x201d: '"',
    # ligatures
    # from https://en.wikipedia.org/wiki/Typographic_ligature#Ligatures_in_Unicode_.28Latin_alphabets.29  # noqa
    0xfb00: 'ff',
    0xfb01: 'fi',
    0xfb02: 'fl',
    0xfb03: 'ffi',
    0xfb04: 'ffl',
    0xfb06: 'st',
}


def clean_string(s):
    """Clean messy strings from the outside world."""
    if not isinstance(s, str):
        raise TypeError

    s = unicodedata.normalize('NFKD', s)
    s = s.translate(BAD_CODEPOINTS)
    s = simplify_whitespace(s)

    return s


def simplify_whitespace(s):
    """Strip s, and use only single spaces within s."""
    return WHITESPACE_RE.sub(' ', s.strip())


def to_title_case(s):
    """Like titlecase.titlecase(), but treat hyphens as spaces."""
    return ''.join(
        '-' if s[i] == '-' else c
        for i, c in enumerate(titlecase(s.replace('-', ' '))))


def norm(s):
    """Remove accents and convert to lowercase."""
    return unidecode(s).lower()


def smunch(s):
    """Like norm(), except we remove whitespace too."""
    return WHITESPACE_RE.sub('', norm(s))
{'fraction_non_alphanumeric': 0.13087984401657324, 'fraction_numerical': 0.025834755057275165, 'mean_word_length': 3.4950711938663748, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2911966', 'n_tokens_mistral': 1412, 'n_tokens_neox': 1328, 'n_words': 435}