<commit_before>#
# BSD 3-Clause License
#
# Copyright (c) 2017, plures
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its
#    contributors may be used to endorse or promote products derived from
#    this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

import sys, argparse
import unittest, gc
import weakref, struct
from copy import copy
from ndtypes import ndt, typedef, MAX_DIM
from randtype import *


HAVE_PYTHON_36 = sys.version_info >= (3, 6, 0)
HAVE_32_BIT_LINUX = sys.maxsize == 2**31-1 and sys.platform == 'linux'


class TestModule(unittest.TestCase):

    def test_module_predicates(self):
        # Namespaces are not yet supported in xnd. One can construct the
        # types, however.  Modules are for pattern matching only, so they 
        # are abstract.
        t = ndt("SomeNamespace:: 2 * 3 * float64")

        self.assertTrue(t.isabstract())
        self.assertFalse(t.iscomplex())
        self.assertFalse(t.isconcrete())
        self.assertFalse(t.isfloat())
        self.assertFalse(t.isoptional())
        self.assertFalse(t.isscalar())
        self.assertFalse(t.issigned())
        self.assertFalse(t.isunsigned())

        self.assertFalse(t.is_f_contiguous())
        self.assertFalse(t.is_c_contiguous())

    def test_module_common_fields(self):
        t = ndt("SomeNamespace:: 2 * 3 * float64")

        # Common type fields are undefined.
        self.assertRaises(TypeError, getattr, t, 'ndim')
        self.assertRaises(TypeError, getattr, t, 'itemsize')
        self.assertRaises(TypeError, getattr, t, 'align')

        # Cannot be represented as an ndarray.
        self.assertRaises(TypeError, getattr, t, 'shape')
        self.assertRaises(TypeError, getattr, t, 'strides')


class TestFunction(unittest.TestCase):

    def test_function_predicates(self):
        t = ndt("(10 * float64, string) -> float64")
 
        self.assertTrue(t.isabstract())
        self.assertFalse(t.iscomplex())
        self.assertFalse(t.isconcrete())
        self.assertFalse(t.isfloat())
        self.assertFalse(t.isoptional())
        self.assertFalse(t.isscalar())
        self.assertFalse(t.issigned())
        self.assertFalse(t.isunsigned())

        self.assertFalse(t.is_c_contiguous())
        self.assertFalse(t.is_f_contiguous())

    def test_function_common_fields(self):
        t = ndt("(10 * float64, string) -> float64")

        # Common type fields are undefined.
        self.assertRaises(TypeError, getattr, t, 'ndim')
        self.assertRaises(TypeError, getattr, t, 'itemsize')
        self.assertRaises(TypeError, getattr, t, 'align')

        # Cannot be represented as an ndarray.
        self.assertRaises(TypeError, getattr, t, 'shape')
        self.assertRaises(TypeError, getattr, t, 'strides')


class TestVoid(unittest.TestCase):

    def test_void_as_return_value(self):
        t = ndt("() -> void")

        self.assertTrue(t.isabstract())
        self.assertFalse(t.iscomplex())
        self.assertFalse(t.isconcrete())
        self.assertFalse(t.isfloat())
        self.assertFalse(t.isoptional())
        self.assertFalse(t.isscalar())
        self.assertFalse(t.issigned())
        self.assertFalse(t.isunsigned())

        self.assertFalse(t.is_c_contiguous())
        self.assertFalse(t.is_f_contiguous())

    def test_void_exceptions(self):
        # Void can only be used as a function return type.
        self.assertRaises(ValueError, ndt, "void")


class TestAny(unittest.TestCase):

    def test_any_predicates(self):
        t = ndt("Any")

        self.assertTrue(t.isabstract())
        self.assertFalse(t.iscomplex())
        self.assertFalse(t.isconcrete())
        self.assertFalse(t.isfloat())
        self.assertFalse(t.isoptional())
        self.assertFalse(t.isscalar())
        self.assertFalse(t.issigned())
        self.assertFalse(t.isunsigned())

        self.assertFalse(t.is_c_contiguous())
        self.assertFalse(t.is_f_contiguous())

    def test_any_common_fields(self):
        t = ndt("Any")

        # Common type fields are undefined.
        self.assertRaises(TypeError, getattr, t, 'ndim')
        self.assertRaises(TypeError, getattr, t, 'itemsize')
        self.assertRaises(TypeError, getattr, t, 'align')

        # Cannot be represented as an ndarray.
        self.assertRaises(TypeError, getattr, t, 'shape')
        self.assertRaises(TypeError, getattr, t, 'strides')


class TestFixedDim(unittest.TestCase):

    def test_fixed_dim_predicates(self):
        t = ndt("10 * 20 * complex128")

        self.assertFalse(t.isabstract())
        self.assertFalse(t.iscomplex())
        self.assertTrue(t.isconcrete())
        self.assertFalse(t.isfloat())
        self.assertFalse(t.isoptional())
        self.assertFalse(t.isscalar())
        self.assertFalse(t.issigned())
        self.assertFalse(t.isunsigned())

        self.assertTrue(t.is_c_contiguous())
        self.assertFalse(t.is_f_contiguous())

        t = ndt("20 * complex128")
        self.assertTrue(t.is_c_contiguous())
        self.assertTrue(t.is_f_contiguous())

        t = ndt("1 * 10 * complex128")
        self.assertTrue(t.is_c_contiguous())
        self.assertTrue(t.is_f_contiguous())

    def test_fixed_dim_common_fields(self):
        dt = "{a: complex64, b: string}"
        t = ndt("2 * 3 * %s" % dt)
        dtype = ndt(dt)

        self.assertEqual(t.ndim, 2)
        if not HAVE_32_BIT_LINUX:
            self.assertEqual(t.itemsize, dtype.itemsize)
            self.assertEqual(t.align, dtype.align)

        self.assertEqual(t.shape, (2, 3))
        self.assertEqual(t.strides, (3 * dtype.itemsize, dtype.itemsize))

    def test_fixed_dim_invariants(self):
        # Mixing var and fixed is disallowed.
        self.assertRaises(TypeError, ndt, "10 * var * int8")
        self.assertRaises(TypeError, ndt, "var * 10 * int16")
        self.assertRaises(TypeError, ndt, "10 * var * 10 * int32")
        self.assertRaises(TypeError, ndt, "var * 10 * var * int64")

    def test_fixed_dim_dtypes(self):
        for dtype, mem in DTYPE_TEST_CASES:
            t = ndt(dtype)
            self.assertEqual(t.ndim, 0)
            if not HAVE_32_BIT_LINUX:
                self.assertEqual(t.itemsize, mem.itemsize)
                self.assertEqual(t.align, mem.align)

            self.assertEqual(t.shape, ())
            self.assertEqual(t.strides, ())

            for i in range(10):
                t = ndt("%d * %s" % (i, dtype))
                shape = (i,)
                strides = (mem.itemsize,)

                self.assertEqual(t.ndim, 1)
                if not HAVE_32_BIT_LINUX:
                    self.assertEqual(t.itemsize, mem.itemsize)
                    self.assertEqual(t.align, mem.align)
                    self.assertEqual(t.strides, strides)

                self.assertEqual(t.shape, shape)

            for i in range(10):
                for j in range(10):
                    t = ndt("%d * %d * %s" % (i, j, dtype))
                    shape = (i, j)
                    strides = (j * mem.itemsize, mem.itemsize)

                    self.assertEqual(t.ndim, 2)
                    if not HAVE_32_BIT_LINUX:
                        self.assertEqual(t.itemsize, mem.itemsize)
                        self.assertEqual(t.align, mem.align)
                        self.assertEqual(t.strides, strides)

                    self.assertEqual(t.shape, shape)

            for i in range(5):
                for j in range(5):
                    for k in range(5):
                        t = ndt("%d * %d * %d * %s" % (i, j, k, dtype))
                        shape = (i, j, k)
                        strides = (j * k * mem.itemsize, k * mem.itemsize, mem.itemsize)

                        self.assertEqual(t.ndim, 3)
                        if not HAVE_32_BIT_LINUX:
                            self.assertEqual(t.itemsize, mem.itemsize)
                            self.assertEqual(t.align, mem.align)
                            self.assertEqual(t.strides, strides)

                        self.assertEqual(t.shape, shape)


class TestFortran(unittest.TestCase):

    def test_fortran_predicates(self):
        t = ndt("!10 * 20 * complex128")

        self.assertFalse(t.isabstract())
        self.assertFalse(t.iscomplex())
        self.assertTrue(t.isconcrete())
        self.assertFalse(t.isfloat())
        self.assertFalse(t.isoptional())
        self.assertFalse(t.isscalar())
        self.assertFalse(t.issigned())
        self.assertFalse(t.isunsigned())

        self.assertFalse(t.is_c_contiguous())
        self.assertTrue(t.is_f_contiguous())

        t = ndt("!20 * complex128")
        self.assertTrue(t.is_c_contiguous())
        self.assertTrue(t.is_f_contiguous())

        t = ndt("!1 * 10 * uint8")
        self.assertTrue(t.is_c_contiguous())
        self.assertTrue(t.is_f_contiguous())

    def test_fortran_common_fields(self):
        dt = "{a: complex64, b: string}"
        t = ndt("!2 * 3 * %s" % dt)
        dtype = ndt(dt)

        self.assertEqual(t.ndim, 2)
        if not HAVE_32_BIT_LINUX:
            self.assertEqual(t.itemsize, dtype.itemsize)
            self.assertEqual(t.align, dtype.align)

        self.assertEqual(t.shape, (2, 3))
        self.assertEqual(t.strides, (dtype.itemsize, 2 * dtype.itemsize))

    def test_fortran_dtypes(self):
        for dtype, mem in DTYPE_TEST_CASES:
            t = ndt(dtype)
            self.assertEqual(t.ndim, 0)
            if not HAVE_32_BIT_LINUX:
                self.assertEqual(t.itemsize, mem.itemsize)
                self.assertEqual(t.align, mem.align)

            self.assertEqual(t.shape, ())
            self.assertEqual(t.strides, ())

            for i in range(10):
                t = ndt("!%d * %s" % (i, dtype))
                shape = (i,)
                strides = (mem.itemsize,)

                self.assertEqual(t.ndim, 1)
                if not HAVE_32_BIT_LINUX:
                    self.assertEqual(t.itemsize, mem.itemsize)
                    self.assertEqual(t.align, mem.align)
                    self.assertEqual(t.strides, strides)

                self.assertEqual(t.shape, shape)

            for i in range(10):
                for j in range(10):
                    t = ndt("!%d * %d * %s" % (i, j, dtype))
                    shape = (i, j)
                    strides = (mem.itemsize, i * mem.itemsize)

                    self.assertEqual(t.ndim, 2)
                    if not HAVE_32_BIT_LINUX:
                        self.assertEqual(t.itemsize, mem.itemsize)
                        self.assertEqual(t.align, mem.align)
                        self.assertEqual(t.strides, strides)

                    self.assertEqual(t.shape, shape)

            for i in range(5):
                for j in range(5):
                    for k in range(5):
                        t = ndt("!%d * %d * %d * %s" % (i, j, k, dtype))
                        shape = (i, j, k)
                        strides = (mem.itemsize, i * mem.itemsize, i * j * mem.itemsize)

                        self.assertEqual(t.ndim, 3)
                        if not HAVE_32_BIT_LINUX:
                            self.assertEqual(t.itemsize, mem.itemsize)
                            self.assertEqual(t.align, mem.align)
                            self.assertEqual(t.strides, strides)

                        self.assertEqual(t.shape, shape)
                        self.assertEqual(t.strides, strides)


class TestVarDim(unittest.TestCase):

    def test_var_dim_predicates(self):<commit_msg>Remove one more test.<commit_after>#
# BSD 3-Clause License
#
# Copyright (c) 2017, plures
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its
#    contributors may be used to endorse or promote products derived from
#    this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#

import sys, argparse
import unittest, gc
import weakref, struct
from copy import copy
from ndtypes import ndt, typedef, MAX_DIM
from randtype import *


HAVE_PYTHON_36 = sys.version_info >= (3, 6, 0)
HAVE_32_BIT_LINUX = sys.maxsize == 2**31-1 and sys.platform == 'linux'


class TestModule(unittest.TestCase):

    def test_module_predicates(self):
        # Namespaces are not yet supported in xnd. One can construct the
        # types, however.  Modules are for pattern matching only, so they 
        # are abstract.
        t = ndt("SomeNamespace:: 2 * 3 * float64")

        self.assertTrue(t.isabstract())
        self.assertFalse(t.iscomplex())
        self.assertFalse(t.isconcrete())
        self.assertFalse(t.isfloat())
        self.assertFalse(t.isoptional())
        self.assertFalse(t.isscalar())
        self.assertFalse(t.issigned())
        self.assertFalse(t.isunsigned())

        self.assertFalse(t.is_f_contiguous())
        self.assertFalse(t.is_c_contiguous())

    def test_module_common_fields(self):
        t = ndt("SomeNamespace:: 2 * 3 * float64")

        # Common type fields are undefined.
        self.assertRaises(TypeError, getattr, t, 'ndim')
        self.assertRaises(TypeError, getattr, t, 'itemsize')
        self.assertRaises(TypeError, getattr, t, 'align')

        # Cannot be represented as an ndarray.
        self.assertRaises(TypeError, getattr, t, 'shape')
        self.assertRaises(TypeError, getattr, t, 'strides')


class TestFunction(unittest.TestCase):

    def test_function_predicates(self):
        t = ndt("(10 * float64, string) -> float64")
 
        self.assertTrue(t.isabstract())
        self.assertFalse(t.iscomplex())
        self.assertFalse(t.isconcrete())
        self.assertFalse(t.isfloat())
        self.assertFalse(t.isoptional())
        self.assertFalse(t.isscalar())
        self.assertFalse(t.issigned())
        self.assertFalse(t.isunsigned())

        self.assertFalse(t.is_c_contiguous())
        self.assertFalse(t.is_f_contiguous())

    def test_function_common_fields(self):
        t = ndt("(10 * float64, string) -> float64")

        # Common type fields are undefined.
        self.assertRaises(TypeError, getattr, t, 'ndim')
        self.assertRaises(TypeError, getattr, t, 'itemsize')
        self.assertRaises(TypeError, getattr, t, 'align')

        # Cannot be represented as an ndarray.
        self.assertRaises(TypeError, getattr, t, 'shape')
        self.assertRaises(TypeError, getattr, t, 'strides')


class TestVoid(unittest.TestCase):

    def test_void_as_return_value(self):
        t = ndt("() -> void")

        self.assertTrue(t.isabstract())
        self.assertFalse(t.iscomplex())
        self.assertFalse(t.isconcrete())
        self.assertFalse(t.isfloat())
        self.assertFalse(t.isoptional())
        self.assertFalse(t.isscalar())
        self.assertFalse(t.issigned())
        self.assertFalse(t.isunsigned())

        self.assertFalse(t.is_c_contiguous())
        self.assertFalse(t.is_f_contiguous())

    def test_void_exceptions(self):
        # Void can only be used as a function return type.
        self.assertRaises(ValueError, ndt, "void")


class TestAny(unittest.TestCase):

    def test_any_predicates(self):
        t = ndt("Any")

        self.assertTrue(t.isabstract())
        self.assertFalse(t.iscomplex())
        self.assertFalse(t.isconcrete())
        self.assertFalse(t.isfloat())
        self.assertFalse(t.isoptional())
        self.assertFalse(t.isscalar())
        self.assertFalse(t.issigned())
        self.assertFalse(t.isunsigned())

        self.assertFalse(t.is_c_contiguous())
        self.assertFalse(t.is_f_contiguous())

    def test_any_common_fields(self):
        t = ndt("Any")

        # Common type fields are undefined.
        self.assertRaises(TypeError, getattr, t, 'ndim')
        self.assertRaises(TypeError, getattr, t, 'itemsize')
        self.assertRaises(TypeError, getattr, t, 'align')

        # Cannot be represented as an ndarray.
        self.assertRaises(TypeError, getattr, t, 'shape')
        self.assertRaises(TypeError, getattr, t, 'strides')


class TestFixedDim(unittest.TestCase):

    def test_fixed_dim_predicates(self):
        t = ndt("10 * 20 * complex128")

        self.assertFalse(t.isabstract())
        self.assertFalse(t.iscomplex())
        self.assertTrue(t.isconcrete())
        self.assertFalse(t.isfloat())
        self.assertFalse(t.isoptional())
        self.assertFalse(t.isscalar())
        self.assertFalse(t.issigned())
        self.assertFalse(t.isunsigned())

        self.assertTrue(t.is_c_contiguous())
        self.assertFalse(t.is_f_contiguous())

        t = ndt("20 * complex128")
        self.assertTrue(t.is_c_contiguous())
        self.assertTrue(t.is_f_contiguous())

        t = ndt("1 * 10 * complex128")
        self.assertTrue(t.is_c_contiguous())
        self.assertTrue(t.is_f_contiguous())

    def test_fixed_dim_common_fields(self):
        dt = "{a: complex64, b: string}"
        t = ndt("2 * 3 * %s" % dt)
        dtype = ndt(dt)

        self.assertEqual(t.ndim, 2)
        if not HAVE_32_BIT_LINUX:
            self.assertEqual(t.itemsize, dtype.itemsize)
            self.assertEqual(t.align, dtype.align)

        self.assertEqual(t.shape, (2, 3))
        self.assertEqual(t.strides, (3 * dtype.itemsize, dtype.itemsize))

    def test_fixed_dim_invariants(self):
        # Mixing var and fixed is disallowed.
        self.assertRaises(TypeError, ndt, "10 * var * int8")
        self.assertRaises(TypeError, ndt, "var * 10 * int16")
        self.assertRaises(TypeError, ndt, "10 * var * 10 * int32")
        self.assertRaises(TypeError, ndt, "var * 10 * var * int64")

    def test_fixed_dim_dtypes(self):
        for dtype, mem in DTYPE_TEST_CASES:
            t = ndt(dtype)
            self.assertEqual(t.ndim, 0)
            if not HAVE_32_BIT_LINUX:
                self.assertEqual(t.itemsize, mem.itemsize)
                self.assertEqual(t.align, mem.align)

            self.assertEqual(t.shape, ())
            self.assertEqual(t.strides, ())

            for i in range(10):
                t = ndt("%d * %s" % (i, dtype))
                shape = (i,)
                strides = (mem.itemsize,)

                self.assertEqual(t.ndim, 1)
                if not HAVE_32_BIT_LINUX:
                    self.assertEqual(t.itemsize, mem.itemsize)
                    self.assertEqual(t.align, mem.align)
                    self.assertEqual(t.strides, strides)

                self.assertEqual(t.shape, shape)

            for i in range(10):
                for j in range(10):
                    t = ndt("%d * %d * %s" % (i, j, dtype))
                    shape = (i, j)
                    strides = (j * mem.itemsize, mem.itemsize)

                    self.assertEqual(t.ndim, 2)
                    if not HAVE_32_BIT_LINUX:
                        self.assertEqual(t.itemsize, mem.itemsize)
                        self.assertEqual(t.align, mem.align)
                        self.assertEqual(t.strides, strides)

                    self.assertEqual(t.shape, shape)

            for i in range(5):
                for j in range(5):
                    for k in range(5):
                        t = ndt("%d * %d * %d * %s" % (i, j, k, dtype))
                        shape = (i, j, k)
                        strides = (j * k * mem.itemsize, k * mem.itemsize, mem.itemsize)

                        self.assertEqual(t.ndim, 3)
                        if not HAVE_32_BIT_LINUX:
                            self.assertEqual(t.itemsize, mem.itemsize)
                            self.assertEqual(t.align, mem.align)
                            self.assertEqual(t.strides, strides)

                        self.assertEqual(t.shape, shape)


class TestFortran(unittest.TestCase):

    def test_fortran_predicates(self):
        t = ndt("!10 * 20 * complex128")

        self.assertFalse(t.isabstract())
        self.assertFalse(t.iscomplex())
        self.assertTrue(t.isconcrete())
        self.assertFalse(t.isfloat())
        self.assertFalse(t.isoptional())
        self.assertFalse(t.isscalar())
        self.assertFalse(t.issigned())
        self.assertFalse(t.isunsigned())

        self.assertFalse(t.is_c_contiguous())
        self.assertTrue(t.is_f_contiguous())

        t = ndt("!20 * complex128")
        self.assertTrue(t.is_c_contiguous())
        self.assertTrue(t.is_f_contiguous())

        t = ndt("!1 * 10 * uint8")
        self.assertTrue(t.is_c_contiguous())
        self.assertTrue(t.is_f_contiguous())

    def test_fortran_common_fields(self):
        dt = "{a: complex64, b: string}"
        t = ndt("!2 * 3 * %s" % dt)
        dtype = ndt(dt)

        self.assertEqual(t.ndim, 2)
        if not HAVE_32_BIT_LINUX:
            self.assertEqual(t.itemsize, dtype.itemsize)
            self.assertEqual(t.align, dtype.align)

        self.assertEqual(t.shape, (2, 3))
        self.assertEqual(t.strides, (dtype.itemsize, 2 * dtype.itemsize))

    def test_fortran_dtypes(self):
        for dtype, mem in DTYPE_TEST_CASES:
            t = ndt(dtype)
            self.assertEqual(t.ndim, 0)
            if not HAVE_32_BIT_LINUX:
                self.assertEqual(t.itemsize, mem.itemsize)
                self.assertEqual(t.align, mem.align)

            self.assertEqual(t.shape, ())
            self.assertEqual(t.strides, ())

            for i in range(10):
                t = ndt("!%d * %s" % (i, dtype))
                shape = (i,)
                strides = (mem.itemsize,)

                self.assertEqual(t.ndim, 1)
                if not HAVE_32_BIT_LINUX:
                    self.assertEqual(t.itemsize, mem.itemsize)
                    self.assertEqual(t.align, mem.align)
                    self.assertEqual(t.strides, strides)

                self.assertEqual(t.shape, shape)

            for i in range(10):
                for j in range(10):
                    t = ndt("!%d * %d * %s" % (i, j, dtype))
                    shape = (i, j)
                    strides = (mem.itemsize, i * mem.itemsize)

                    self.assertEqual(t.ndim, 2)
                    if not HAVE_32_BIT_LINUX:
                        self.assertEqual(t.itemsize, mem.itemsize)
                        self.assertEqual(t.align, mem.align)
                        self.assertEqual(t.strides, strides)

                    self.assertEqual(t.shape, shape)

            for i in range(5):
                for j in range(5):
                    for k in range(5):
                        t = ndt("!%d * %d * %d * %s" % (i, j, k, dtype))
                        shape = (i, j, k)
                        strides = (mem.itemsize, i * mem.itemsize, i * j * mem.itemsize)

                        self.assertEqual(t.ndim, 3)
                        if not HAVE_32_BIT_LINUX:
                            self.assertEqual(t.itemsize, mem.itemsize)
                            self.assertEqual(t.align, mem.align)
                            self.assertEqual(t.strides, strides)

                        self.assertEqual(t.shape, shape)


class TestVarDim(unittest.TestCase):

    def test_var_dim_predicates(self):
{'fraction_non_alphanumeric': 0.11228590378838126, 'fraction_numerical': 0.011860637509266123, 'mean_word_length': 2.10653254151012, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5073196', 'n_tokens_mistral': 8129, 'n_tokens_neox': 7600, 'n_words': 1778}