<commit_before>
from . import _urlparse as urlparse
from functools import reduce

import six

def iterkeys(d, *a, **kw):
    return iter(getattr(d, six._iterkeys)(*a, **kw))

def itervalues(d, *a, **kw):
    return iter(getattr(d, six._itervalues)(*a, **kw))

def iteritems(d, *a, **kw):
    return iter(getattr(d, six._iteritems)(*a, **kw))

if six.PY3:
    _iterlists = 'lists'
    _iterlistvalues = 'listvalues'
else:
    _iterlists = 'iterlists'
    _iterlistvalues = 'iterlistvalues'

def iterlists(d, *a, **kw):
    return getattr(d, _iterlists)(*a, **kw)

def iterlistvalues(d, *a, **kw):
    return getattr(d, _iterlistvalues)(*a, **kw)

try:
    unichr = unichr # py2
except NameError:
    unichr = chr # py3

def to_bytes(x, charset):
    '''please use carefully'''
    if six.PY3:
        if not isinstance(x, bytes):
            x = str(x).encode(charset)
        return x
    if isinstance(x, unicode):
        return x.encode(charset)
    return str(x)

def to_native(x, charset):
    '''please use carefully'''
    if isinstance(x, str):
        return x
    if six.PY3:
        return x.decode(charset)
    else:
        return x.encode(charset)

def string_join(list):
    '''concatenate any string type'''
    return reduce(lambda x, y: x + y, list)<commit_msg>Add default to string_join; make efficient<commit_after>
from . import _urlparse as urlparse
from functools import reduce

import six

def iterkeys(d, *a, **kw):
    return iter(getattr(d, six._iterkeys)(*a, **kw))

def itervalues(d, *a, **kw):
    return iter(getattr(d, six._itervalues)(*a, **kw))

def iteritems(d, *a, **kw):
    return iter(getattr(d, six._iteritems)(*a, **kw))

if six.PY3:
    _iterlists = 'lists'
    _iterlistvalues = 'listvalues'
else:
    _iterlists = 'iterlists'
    _iterlistvalues = 'iterlistvalues'

def iterlists(d, *a, **kw):
    return getattr(d, _iterlists)(*a, **kw)

def iterlistvalues(d, *a, **kw):
    return getattr(d, _iterlistvalues)(*a, **kw)

try:
    unichr = unichr # py2
except NameError:
    unichr = chr # py3

def to_bytes(x, charset):
    '''please use carefully'''
    if six.PY3:
        if not isinstance(x, bytes):
            x = str(x).encode(charset)
        return x
    if isinstance(x, unicode):
        return x.encode(charset)
    return str(x)

def to_native(x, charset):
    '''please use carefully'''
    if isinstance(x, str):
        return x
    if six.PY3:
        return x.decode(charset)
    else:
        return x.encode(charset)

def string_join(list, default=''):
    '''concatenate any string type'''
    if list:
        if isinstance(list[0], bytes):
            return b''.join(list)
        return u''.join(list)
    return default
{'fraction_non_alphanumeric': 0.14572490706319702, 'fraction_numerical': 0.004089219330855019, 'mean_word_length': 3.1527777777777777, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7349653', 'n_tokens_mistral': 976, 'n_tokens_neox': 956, 'n_words': 261}