<commit_before>import re
from datetime import datetime
from time import mktime, gmtime


def documentUrl(x):
    "Return the original document URL from the URL in the document version"
    i = x.find('churl')
    return 'http://' + x[x.find('/', i + 6) + 1:]


def countryUri(x):
    "Return a URI for a country given its name."
    import re

    x = re.sub('[^A-Za-z0-9]+', '', x)
    return x.lower()


def personNameUri(x):
    "Return a URI for a person name."
    import re

    x = re.sub('[^A-Za-z0-9]+', '', x.strip())
    return x.lower()


def toTitleCaseIfUpper(x):
    "Return the string in title case if it is all upper, otherwise leave capitalization alone."
    x = x.strip()
    if x.isupper():
        return x.title()
    else:
        return x

def nonWhitespace(x):
    "Return the string cleaning spaces."
    import re

    y = re.sub(r'\s+', ' ', x.strip())
    return y

def toTitleCaseCleaned(x):
    "Return the string in title case cleaning spaces."
    y = nonWhitespace(x)
    return y.title()

def nonAsciiChars(x):
    "Return a set of the non-ascii chars in x"
    import re

    return set(re.sub('[\x00-\x7f]', '', x))<commit_msg>DIG-102 @comment remove spaces from email addresses<commit_after>import re
from datetime import datetime
from time import mktime, gmtime


def documentUrl(x):
    "Return the original document URL from the URL in the document version"
    i = x.find('churl')
    return 'http://' + x[x.find('/', i + 6) + 1:]


def countryUri(x):
    "Return a URI for a country given its name."
    import re

    x = re.sub('[^A-Za-z0-9]+', '', x)
    return x.lower()


def personNameUri(x):
    "Return a URI for a person name."
    import re

    x = re.sub('[^A-Za-z0-9]+', '', x.strip())
    return x.lower()


def toTitleCaseIfUpper(x):
    "Return the string in title case if it is all upper, otherwise leave capitalization alone."
    x = x.strip()
    if x.isupper():
        return x.title()
    else:
        return x

def nonWhitespace(x):
    "Return the string removing all spaces."
    import re

    y = re.sub(r'\s+', '', x.strip())
    return y

def toTitleCaseCleaned(x):
    "Return the string in title case cleaning spaces."
    import re

    y = re.sub(r'\s+', ' ', x.strip())
    return y.title()

def nonAsciiChars(x):
    "Return a set of the non-ascii chars in x"
    import re

    return set(re.sub('[\x00-\x7f]', '', x))
{'fraction_non_alphanumeric': 0.14083333333333334, 'fraction_numerical': 0.00875, 'mean_word_length': 2.699537750385208, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5790661', 'n_tokens_mistral': 848, 'n_tokens_neox': 803, 'n_words': 283}