<commit_before># -*- coding: utf-8 -*-

# Futures #
from __future__ import division

# Built-in modules #
import sys, os, time, re, random, math, json
import getpass, hashlib, collections
import unicodedata
from itertools import compress, product

# Third party modules #
from six import string_types

# One liners #
flatter = lambda x: [item for sublist in x for item in sublist]

################################################################################
def ascii(text):
    """Make a safe, ASCII version a string. For instance for use on the web."""
    return unicodedata.normalize('NFKD', unicode(text)).encode('ASCII', 'ignore')

def alphanumeric(text):
    r"""Make an ultra-safe, ASCII version a string.
    For instance for use as a filename.
    \w matches any alphanumeric character and the underscore."""
    return "".join([c for c in text if re.match(r'\w', c)])

################################################################################
def sanitize_text(text):
    r"""Make a safe representation of a string.
    Note: the `\s` special character matches any whitespace character.
    This is equivalent to the set [\t\n\r\f\v] as well as ` ` (whitespace)."""
    # First replace characters that have specific effects with their repr #
    text = re.sub("(\s)", lambda m: repr(m.group(0)).strip("'"), text)
    # Make it a unicode string (the try supports python 2 and 3) #
    try: text = text.decode('utf-8')
    except AttributeError: pass
    # Normalize it “
    text = unicodedata.normalize('NFC', text)
    return text

################################################################################
def camel_to_snake(text):
    """
    Will convert CamelCaseStrings to snake_case_strings.
    >>> camel_to_snake('CamelCase')
    'camel_case'
    >>> camel_to_snake('CamelCamelCase')
    'camel_camel_case'
    >>> camel_to_snake('Camel2Camel2Case')
    'camel2_camel2_case'
    >>> camel_to_snake('getHTTPResponseCode')
    'get_http_response_code'
    >>> camel_to_snake('get2HTTPResponseCode')
    'get2_http_response_code'
    >>> camel_to_snake('HTTPResponseCode')
    'http_response_code'
    >>> camel_to_snake('HTTPResponseCodeXYZ')
    'http_response_code_xyz'
    >>> camel_to_snake('Double_Case')
    'double_case'
    >>> camel_to_snake('Odd/Characters')
    'odd_characters'
    """
    # Two step process #
    result = re.sub('(.)([A-Z][a-z]+)',  r'\1_\2', text)
    result = re.sub('([a-z0-9])([A-Z])', r'\1_\2', result)
    result = result.lower()
    # Eliminate spaces #
    result = result.replace(' ', '')
    # Eliminate quote characters #
    result = result.replace('"', '')
    result = result.replace("'", '')
    # Eliminate special characters #
    result = result.replace('/', '_')
    # Eliminate double underscore #
    result = result.replace('__', '_')
    # Return #
    return result

################################################################################
def bool_to_unicode(b):
    """Different possibilities for True: ☑️✔︎✓✅👍✔️
       Different possibilities for False: ✕✖︎✗✘✖️❌⛔️❎👎🛑🔴"""
    b = bool(b)
    if b is True:  return u"✅"
    if b is False: return u"❎"

###############################################################################
def access_dict_like_obj(obj, prop, new_value=None):
    """
    Access a dictionary like if it was an object with properties.
    If no "new_value", then it's a getter, otherwise it's a setter.
    >>> {'characters': {'cast': '<NAME>', 'featuring': '<NAME>'}}
    >>> access_dict_like_obj(startrek, 'characters.cast', '<NAME>')
    """
    props = prop.split('.')
    if new_value:
        if props[0] not in obj: obj[props[0]] = {}
        if len(props)==1: obj[prop] = new_value
        else: return access_dict_like_obj(obj[props[0]], '.'.join(props[1:]), new_value)
    else:
        if len(props)==1: return obj[prop]
        else: return access_dict_like_obj(obj[props[0]], '.'.join(props[1:]))<commit_msg>Border case in camel_to_snake function<commit_after># -*- coding: utf-8 -*-

# Futures #
from __future__ import division

# Built-in modules #
import sys, os, time, re, random, math, json
import getpass, hashlib, collections
import unicodedata
from itertools import compress, product

# Third party modules #
from six import string_types

# One liners #
flatter = lambda x: [item for sublist in x for item in sublist]

################################################################################
def ascii(text):
    """Make a safe, ASCII version a string. For instance for use on the web."""
    return unicodedata.normalize('NFKD', unicode(text)).encode('ASCII', 'ignore')

def alphanumeric(text):
    r"""Make an ultra-safe, ASCII version a string.
    For instance for use as a filename.
    \w matches any alphanumeric character and the underscore."""
    return "".join([c for c in text if re.match(r'\w', c)])

################################################################################
def sanitize_text(text):
    r"""Make a safe representation of a string.
    Note: the `\s` special character matches any whitespace character.
    This is equivalent to the set [\t\n\r\f\v] as well as ` ` (whitespace)."""
    # First replace characters that have specific effects with their repr #
    text = re.sub("(\s)", lambda m: repr(m.group(0)).strip("'"), text)
    # Make it a unicode string (the try supports python 2 and 3) #
    try: text = text.decode('utf-8')
    except AttributeError: pass
    # Normalize it “
    text = unicodedata.normalize('NFC', text)
    return text

################################################################################
def camel_to_snake(text):
    """
    Will convert CamelCaseStrings to snake_case_strings.
    >>> camel_to_snake('CamelCase')
    'camel_case'
    >>> camel_to_snake('CamelCamelCase')
    'camel_camel_case'
    >>> camel_to_snake('Camel2Camel2Case')
    'camel2_camel2_case'
    >>> camel_to_snake('getHTTPResponseCode')
    'get_http_response_code'
    >>> camel_to_snake('get2HTTPResponseCode')
    'get2_http_response_code'
    >>> camel_to_snake('HTTPResponseCode')
    'http_response_code'
    >>> camel_to_snake('HTTPResponseCodeXYZ')
    'http_response_code_xyz'
    >>> camel_to_snake('Double_Case')
    'double_case'
    >>> camel_to_snake('SW_Merch')
    'double_case'
    >>> camel_to_snake('Odd/Characters')
    'odd_characters'
    """
    # Two step process #
    result = re.sub('(.)([A-Z][a-z]+)',  r'\1_\2', text)
    result = re.sub('([a-z0-9])([A-Z])', r'\1_\2', result)
    result = result.lower()
    # Eliminate spaces #
    result = result.replace(' ', '')
    # Eliminate quote characters #
    result = result.replace('"', '')
    result = result.replace("'", '')
    # Eliminate special characters #
    result = result.replace('/', '_')
    # Eliminate double underscore #
    while '__' in result: result = result.replace('__', '_')
    # Return #
    return result

################################################################################
def bool_to_unicode(b):
    """Different possibilities for True: ☑️✔︎✓✅👍✔️
       Different possibilities for False: ✕✖︎✗✘✖️❌⛔️❎👎🛑🔴"""
    b = bool(b)
    if b is True:  return u"✅"
    if b is False: return u"❎"

###############################################################################
def access_dict_like_obj(obj, prop, new_value=None):
    """
    Access a dictionary like if it was an object with properties.
    If no "new_value", then it's a getter, otherwise it's a setter.
    >>> {'characters': {'cast': '<NAME>', 'featuring': '<NAME>'}}
    >>> access_dict_like_obj(startrek, 'characters.cast', '<NAME>')
    """
    props = prop.split('.')
    if new_value:
        if props[0] not in obj: obj[props[0]] = {}
        if len(props)==1: obj[prop] = new_value
        else: return access_dict_like_obj(obj[props[0]], '.'.join(props[1:]), new_value)
    else:
        if len(props)==1: return obj[prop]
        else: return access_dict_like_obj(obj[props[0]], '.'.join(props[1:]))
{'fraction_non_alphanumeric': 0.23673673673673673, 'fraction_numerical': 0.006256256256256257, 'mean_word_length': 4.046085858585859, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 78, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '188152', 'n_tokens_mistral': 2571, 'n_tokens_neox': 2560, 'n_words': 654}