<commit_before>import sys

from numpy.testing import *
from numpy.core import *
from numpy.compat import asbytes

# Guess the UCS length for this python interpreter
if sys.version_info[0] >= 3:
    import array as _array
    ucs4 = (_array.array('u').itemsize == 4)
    def buffer_length(arr):
        if isinstance(arr, unicode):
            return _array.array('u').itemsize * len(arr)
        v = memoryview(arr)
        if v.shape is None:
            return len(v) * v.itemsize
        else:
            return prod(v.shape) * v.itemsize
else:
    if len(buffer(u'u')) == 4:
        ucs4 = True
    else:
        ucs4 = False
    def buffer_length(arr):
        if isinstance(arr, ndarray):
            return len(arr.data)
        return len(buffer(arr))

# In both cases below we need to make sure that the byte swapped value (as<commit_msg>Fix unicode tests for flexible string representation on Python 3.3<commit_after>import sys

from numpy.testing import *
from numpy.core import *
from numpy.compat import asbytes

# Guess the UCS length for this python interpreter
if sys.version_info[:2] >= (3, 3):
    # Python 3.3 uses a flexible string representation
    ucs4 = False
    def buffer_length(arr):
        if isinstance(arr, unicode):
            arr = str(arr)
            return (sys.getsizeof(arr+"a") - sys.getsizeof(arr)) * len(arr)
        v = memoryview(arr)
        if v.shape is None:
            return len(v) * v.itemsize
        else:
            return prod(v.shape) * v.itemsize
elif sys.version_info[0] >= 3:
    import array as _array
    ucs4 = (_array.array('u').itemsize == 4)
    def buffer_length(arr):
        if isinstance(arr, unicode):
            return _array.array('u').itemsize * len(arr)
        v = memoryview(arr)
        if v.shape is None:
            return len(v) * v.itemsize
        else:
            return prod(v.shape) * v.itemsize
else:
    if len(buffer(u'u')) == 4:
        ucs4 = True
    else:
        ucs4 = False
    def buffer_length(arr):
        if isinstance(arr, ndarray):
            return len(arr.data)
        return len(buffer(arr))

# In both cases below we need to make sure that the byte swapped value (as
{'fraction_non_alphanumeric': 0.10362219165520403, 'fraction_numerical': 0.010087116001834021, 'mean_word_length': 2.326219512195122, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5373854', 'n_tokens_mistral': 680, 'n_tokens_neox': 664, 'n_words': 213}