<commit_before>import sys

from numpy.testing import *
from numpy.core import *

# Guess the UCS length for this python interpreter
if len(buffer(u'u')) == 4:
    ucs4 = True
else:
    ucs4 = False

# Value that can be represented in UCS2 interpreters
ucs2_value = u'\uFFFF'
# Value that cannot be represented in UCS2 interpreters (but can in UCS4)
ucs4_value = u'\U0010FFFF'


############################################################
#    Creation tests
############################################################

class create_zeros(object):
    """Check the creation of zero-valued arrays"""

    def content_check(self, ua, ua_scalar, nbytes):

        # Check the length of the unicode base type
        self.assert_(int(ua.dtype.str[2:]) == self.ulen)
        # Check the length of the data buffer
        self.assert_(len(ua.data) == nbytes)
        # Small check that data in array element is ok
        self.assert_(ua_scalar == u'')
        # Encode to ascii and double check
        self.assert_(ua_scalar.encode('ascii') == '')
        # Check buffer lengths for scalars
        if ucs4:
            self.assert_(len(buffer(ua_scalar)) == 0)
        else:
            self.assert_(len(buffer(ua_scalar)) == 0)
<commit_msg>3K: core: stub out UCS size check on Py3 in test_unicode<commit_after>import sys

from numpy.testing import *
from numpy.core import *

# Guess the UCS length for this python interpreter
if sys.version_info[0] >= 3:
    import warnings
    warnings.warn('XXX: how to detect UCS size on Py3?')
    ucs4 = True
else:
    if len(buffer(u'u')) == 4:
        ucs4 = True
    else:
        ucs4 = False

# Value that can be represented in UCS2 interpreters
ucs2_value = u'\uFFFF'
# Value that cannot be represented in UCS2 interpreters (but can in UCS4)
ucs4_value = u'\U0010FFFF'


############################################################
#    Creation tests
############################################################

class create_zeros(object):
    """Check the creation of zero-valued arrays"""

    def content_check(self, ua, ua_scalar, nbytes):

        # Check the length of the unicode base type
        self.assert_(int(ua.dtype.str[2:]) == self.ulen)
        # Check the length of the data buffer
        self.assert_(len(ua.data) == nbytes)
        # Small check that data in array element is ok
        self.assert_(ua_scalar == u'')
        # Encode to ascii and double check
        self.assert_(ua_scalar.encode('ascii') == '')
        # Check buffer lengths for scalars
        if ucs4:
            self.assert_(len(buffer(ua_scalar)) == 0)
        else:
            self.assert_(len(buffer(ua_scalar)) == 0)

{'fraction_non_alphanumeric': 0.18198874296435272, 'fraction_numerical': 0.01425891181988743, 'mean_word_length': 3.146189735614308, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7637540', 'n_tokens_mistral': 813, 'n_tokens_neox': 774, 'n_words': 252}