<commit_before>import binascii
import hmac
import random
import urllib
from urlparse import urlparse, urlunparse
try:
    from hashlib import sha1
    sha = sha1
except ImportError:
    # hashlib was added in Python 2.5
    import sha

escape = lambda url: urllib.quote(to_utf8(url), safe='~')

def to_utf8(x):
    """
    Tries to utf-8 encode x when possible 

    If x is a string returns it encoded, otherwise tries to iter x and 
    encode utf-8 all strings it contains, returning a list.
    """
    if isinstance(x, basestring): 
        return x.encode('utf-8')
    try:
        l = iter(x)
    except TypeError:
        return x
    return [to_utf8_if_string(i) for i in l]
<commit_msg>Fixing encoding issues, thanks to jqxl0205<commit_after>import binascii
import hmac
import random
import urllib
from urlparse import urlparse, urlunparse
try:
    from hashlib import sha1
    sha = sha1
except ImportError:
    # hashlib was added in Python 2.5
    import sha

escape = lambda url: urllib.quote(to_utf8(url), safe='~')

def to_utf8(x):
    """
    Tries to utf-8 encode x when possible 

    If x is a string returns it encoded, otherwise tries to iter x and 
    encode utf-8 all strings it contains, returning a list.
    """
    if isinstance(x, basestring): 
        return x.encode('utf-8') if isinstance(x, unicode) else x
    try:
        l = iter(x)
    except TypeError:
        return x
    return [to_utf8(i) for i in l]

{'fraction_non_alphanumeric': 0.07756232686980609, 'fraction_numerical': 0.01662049861495845, 'mean_word_length': 2.894878706199461, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6915515', 'n_tokens_mistral': 474, 'n_tokens_neox': 458, 'n_words': 180}