<commit_before># coding: utf8
from __future__ import unicode_literals

import six
import ftfy
import sys
import ujson

try:
    import cPickle as pickle
except ImportError:
    import pickle

try:
    import copy_reg
except ImportError:
    import copyreg as copy_reg

try:
    from cupy.cuda.stream import Stream as CudaStream
except ImportError:
    CudaStream = None

try:
    import cupy
except ImportError:
    cupy = None


pickle = pickle
copy_reg = copy_reg
CudaStream = CudaStream
cupy = cupy
fix_text = ftfy.fix_text

is_python2 = six.PY2
is_python3 = six.PY3
is_windows = sys.platform.startswith('win')
is_linux = sys.platform.startswith('linux')
is_osx = sys.platform == 'darwin'


if is_python2:
    bytes_ = str
    unicode_ = unicode
    basestring_ = basestring
    input_ = raw_input
    json_dumps = lambda data: ujson.dumps(data, indent=2).decode('utf8')
    path2str = lambda path: str(path).decode('utf8')

elif is_python3:
    bytes_ = bytes
    unicode_ = str
    basestring_ = str
    input_ = input
    json_dumps = lambda data: ujson.dumps(data, indent=2)
    path2str = lambda path: str(path)


def symlink_to(orig, dest):
    if is_python2 and is_windows:
        import subprocess
        subprocess.call(['mklink', '/d', path2str(orig), path2str(dest)], shell=True)
    else:
        orig.symlink_to(dest)


def is_config(python2=None, python3=None, windows=None, linux=None, osx=None):
    return ((python2 == None or python2 == is_python2) and
            (python3 == None or python3 == is_python3) and
            (windows == None or windows == is_windows) and
            (linux == None or linux == is_linux) and
            (osx == None or osx == is_osx))<commit_msg>Add compat function to normalize dict keys<commit_after># coding: utf8
from __future__ import unicode_literals

import six
import ftfy
import sys
import ujson

import thinc.neural.util

try:
    import cPickle as pickle
except ImportError:
    import pickle

try:
    import copy_reg
except ImportError:
    import copyreg as copy_reg

try:
    from cupy.cuda.stream import Stream as CudaStream
except ImportError:
    CudaStream = None

try:
    import cupy
except ImportError:
    cupy = None


pickle = pickle
copy_reg = copy_reg
CudaStream = CudaStream
cupy = cupy
fix_text = ftfy.fix_text
copy_array = thinc.neural.util.copy_array

is_python2 = six.PY2
is_python3 = six.PY3
is_windows = sys.platform.startswith('win')
is_linux = sys.platform.startswith('linux')
is_osx = sys.platform == 'darwin'


if is_python2:
    bytes_ = str
    unicode_ = unicode
    basestring_ = basestring
    input_ = raw_input
    json_dumps = lambda data: ujson.dumps(data, indent=2).decode('utf8')
    path2str = lambda path: str(path).decode('utf8')

elif is_python3:
    bytes_ = bytes
    unicode_ = str
    basestring_ = str
    input_ = input
    json_dumps = lambda data: ujson.dumps(data, indent=2)
    path2str = lambda path: str(path)


def symlink_to(orig, dest):
    if is_python2 and is_windows:
        import subprocess
        subprocess.call(['mklink', '/d', path2str(orig), path2str(dest)], shell=True)
    else:
        orig.symlink_to(dest)


def is_config(python2=None, python3=None, windows=None, linux=None, osx=None):
    return ((python2 == None or python2 == is_python2) and
            (python3 == None or python3 == is_python3) and
            (windows == None or windows == is_windows) and
            (linux == None or linux == is_linux) and
            (osx == None or osx == is_osx))


def normalize_string_keys(old):
    '''Given a dictionary, make sure keys are unicode strings, not bytes.'''
    new = {}
    for key, value in old:
        if isinstance(key, bytes_):
            new[key.decode('utf8')] = value
        else:
            new[key] = value
    return new


{'fraction_non_alphanumeric': 0.09520042194092827, 'fraction_numerical': 0.012921940928270042, 'mean_word_length': 3.2285395763656632, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5063024', 'n_tokens_mistral': 1303, 'n_tokens_neox': 1274, 'n_words': 362}