<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, division
import tempfile
import shutil
import atexit


import numpy as np
from nose.tools import eq_ as eq, assert_is_none, assert_is_instance, \
    assert_raises
from numpy.testing import assert_array_equal


from zarr.creation import array, empty, zeros, ones, full, open_array, \
    empty_like, zeros_like, ones_like, full_like, open_like, create
from zarr.sync import ThreadSynchronizer
from zarr.core import Array
from zarr.storage import DirectoryStore
from zarr.hierarchy import open_group
from zarr.errors import PermissionError
from zarr.codecs import Zlib


# something bcolz-like
class MockBcolzArray(object):

    def __init__(self, data, chunklen):
        self.data = data
        self.chunklen = chunklen

    def __getattr__(self, item):
        return getattr(self.data, item)

    def __getitem__(self, item):
        return self.data[item]


# something h5py-like
class MockH5pyDataset(object):

    def __init__(self, data, chunks):
        self.data = data
        self.chunks = chunks

    def __getattr__(self, item):
        return getattr(self.data, item)

    def __getitem__(self, item):
        return self.data[item]


def test_array():

    # with numpy array
    a = np.arange(100)
    z = array(a, chunks=10)
    eq(a.shape, z.shape)
    eq(a.dtype, z.dtype)
    assert_array_equal(a, z[:])

    # with array-like
    a = list(range(100))
    z = array(a, chunks=10)
    eq((100,), z.shape)
    eq(np.asarray(a).dtype, z.dtype)
    assert_array_equal(np.asarray(a), z[:])

    # with another zarr array
    z2 = array(z)
    eq(z.shape, z2.shape)
    eq(z.chunks, z2.chunks)
    eq(z.dtype, z2.dtype)
    assert_array_equal(z[:], z2[:])

    # with chunky array-likes

    b = np.arange(1000).reshape(100, 10)
    c = MockBcolzArray(b, 10)
    z3 = array(c)
    eq(c.shape, z3.shape)
    eq((10, 10), z3.chunks)

    b = np.arange(1000).reshape(100, 10)
    c = MockH5pyDataset(b, chunks=(10, 2))
    z4 = array(c)
    eq(c.shape, z4.shape)
    eq((10, 2), z4.chunks)

    c = MockH5pyDataset(b, chunks=None)
    z5 = array(c)
    eq(c.shape, z5.shape)
    assert_is_instance(z5.chunks, tuple)

    # with dtype=None
    a = np.arange(100, dtype='i4')
    z = array(a, dtype=None)
    assert_array_equal(a[:], z[:])
    eq(a.dtype, z.dtype)

    # with dtype=something else
    a = np.arange(100, dtype='i4')
    z = array(a, dtype='i8')
    assert_array_equal(a[:], z[:])
    eq(np.dtype('i8'), z.dtype)


def test_empty():
    z = empty(100, chunks=10)
    eq((100,), z.shape)
    eq((10,), z.chunks)


def test_zeros():
    z = zeros(100, chunks=10)
    eq((100,), z.shape)
    eq((10,), z.chunks)
    assert_array_equal(np.zeros(100), z[:])


def test_ones():
    z = ones(100, chunks=10)
    eq((100,), z.shape)
    eq((10,), z.chunks)
    assert_array_equal(np.ones(100), z[:])


def test_full():
    z = full(100, chunks=10, fill_value=42, dtype='i4')
    eq((100,), z.shape)
    eq((10,), z.chunks)
    assert_array_equal(np.full(100, fill_value=42, dtype='i4'), z[:])

    # nan
    z = full(100, chunks=10, fill_value=np.nan, dtype='f8')
    assert np.all(np.isnan(z[:]))

    # "NaN" byte string
    v = b'NaN'
    z = full(100, chunks=10, fill_value=v, dtype='S3')
    eq(v, z[0])
    a = z[...]
    print(a.dtype, a[0], a[0:2], repr(a[0]), type(a[0]), a[0] == v, a[0:2] == v)
    eq(v, a[0])
    t = z[...] == v
    assert np.all(t), (np.count_nonzero(t), t.size)

    # "NaN" unicode string
    v = 'NaN'
    z = full(100, chunks=10, fill_value=v, dtype='U3')
    eq(v, z[0])
    a = z[...]
    print(a.dtype, a[0], a[0:2], repr(a[0]), type(a[0]), a[0] == v, a[0:2] == v)
    eq(v, a[0])
    t = z[...] == v
    assert np.all(t), (np.count_nonzero(t), t.size)


def test_open_array():

    store = 'example'

    # mode == 'w'
    z = open_array(store, mode='w', shape=100, chunks=10)
    z[:] = 42
    assert_is_instance(z, Array)
    assert_is_instance(z.store, DirectoryStore)
    eq((100,), z.shape)
    eq((10,), z.chunks)
    assert_array_equal(np.full(100, fill_value=42), z[:])

    # mode in 'r', 'r+'
    open_group('example_group', mode='w')
    for mode in 'r', 'r+':<commit_msg>spike appveyor 5<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, division
import tempfile
import shutil
import atexit


import numpy as np
from nose.tools import eq_ as eq, assert_is_none, assert_is_instance, \
    assert_raises
from numpy.testing import assert_array_equal


from zarr.creation import array, empty, zeros, ones, full, open_array, \
    empty_like, zeros_like, ones_like, full_like, open_like, create
from zarr.sync import ThreadSynchronizer
from zarr.core import Array
from zarr.storage import DirectoryStore
from zarr.hierarchy import open_group
from zarr.errors import PermissionError
from zarr.codecs import Zlib


# something bcolz-like
class MockBcolzArray(object):

    def __init__(self, data, chunklen):
        self.data = data
        self.chunklen = chunklen

    def __getattr__(self, item):
        return getattr(self.data, item)

    def __getitem__(self, item):
        return self.data[item]


# something h5py-like
class MockH5pyDataset(object):

    def __init__(self, data, chunks):
        self.data = data
        self.chunks = chunks

    def __getattr__(self, item):
        return getattr(self.data, item)

    def __getitem__(self, item):
        return self.data[item]


def test_array():

    # with numpy array
    a = np.arange(100)
    z = array(a, chunks=10)
    eq(a.shape, z.shape)
    eq(a.dtype, z.dtype)
    assert_array_equal(a, z[:])

    # with array-like
    a = list(range(100))
    z = array(a, chunks=10)
    eq((100,), z.shape)
    eq(np.asarray(a).dtype, z.dtype)
    assert_array_equal(np.asarray(a), z[:])

    # with another zarr array
    z2 = array(z)
    eq(z.shape, z2.shape)
    eq(z.chunks, z2.chunks)
    eq(z.dtype, z2.dtype)
    assert_array_equal(z[:], z2[:])

    # with chunky array-likes

    b = np.arange(1000).reshape(100, 10)
    c = MockBcolzArray(b, 10)
    z3 = array(c)
    eq(c.shape, z3.shape)
    eq((10, 10), z3.chunks)

    b = np.arange(1000).reshape(100, 10)
    c = MockH5pyDataset(b, chunks=(10, 2))
    z4 = array(c)
    eq(c.shape, z4.shape)
    eq((10, 2), z4.chunks)

    c = MockH5pyDataset(b, chunks=None)
    z5 = array(c)
    eq(c.shape, z5.shape)
    assert_is_instance(z5.chunks, tuple)

    # with dtype=None
    a = np.arange(100, dtype='i4')
    z = array(a, dtype=None)
    assert_array_equal(a[:], z[:])
    eq(a.dtype, z.dtype)

    # with dtype=something else
    a = np.arange(100, dtype='i4')
    z = array(a, dtype='i8')
    assert_array_equal(a[:], z[:])
    eq(np.dtype('i8'), z.dtype)


def test_empty():
    z = empty(100, chunks=10)
    eq((100,), z.shape)
    eq((10,), z.chunks)


def test_zeros():
    z = zeros(100, chunks=10)
    eq((100,), z.shape)
    eq((10,), z.chunks)
    assert_array_equal(np.zeros(100), z[:])


def test_ones():
    z = ones(100, chunks=10)
    eq((100,), z.shape)
    eq((10,), z.chunks)
    assert_array_equal(np.ones(100), z[:])


def test_full():
    z = full(100, chunks=10, fill_value=42, dtype='i4')
    eq((100,), z.shape)
    eq((10,), z.chunks)
    assert_array_equal(np.full(100, fill_value=42, dtype='i4'), z[:])

    # nan
    z = full(100, chunks=10, fill_value=np.nan, dtype='f8')
    assert np.all(np.isnan(z[:]))

    # "NaN" byte string
    v = b'NaN'
    z = full(100, chunks=10, fill_value=v, dtype='S3')
    print(z.store['.zarray'])
    eq(v, z[0])
    a = z[...]
    print(a.dtype, a[0], a[0:2], repr(a[0]), type(a[0]), a[0] == v, a[0:2] == v)
    eq(v, a[0])
    t = z[...] == v
    assert np.all(t), (np.count_nonzero(t), t.size)

    # "NaN" unicode string
    v = 'NaN'
    z = full(100, chunks=10, fill_value=v, dtype='U3')
    print(z.store['.zarray'])
    eq(v, z[0])
    a = z[...]
    print(a.dtype, a[0], a[0:2], repr(a[0]), type(a[0]), a[0] == v, a[0:2] == v)
    eq(v, a[0])
    t = z[...] == v
    assert np.all(t), (np.count_nonzero(t), t.size)


def test_open_array():

    store = 'example'

    # mode == 'w'
    z = open_array(store, mode='w', shape=100, chunks=10)
    z[:] = 42
    assert_is_instance(z, Array)
    assert_is_instance(z.store, DirectoryStore)
    eq((100,), z.shape)
    eq((10,), z.chunks)
    assert_array_equal(np.full(100, fill_value=42), z[:])

    # mode in 'r', 'r+'
    open_group('example_group', mode='w')
    for mode in 'r', 'r+':
{'fraction_non_alphanumeric': 0.17207830266088384, 'fraction_numerical': 0.042785136560778335, 'mean_word_length': 3.0725536992840095, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4910276', 'n_tokens_mistral': 3625, 'n_tokens_neox': 3389, 'n_words': 757}