<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, division
import unittest
import tempfile
import atexit
import pickle
import json
import array
import shutil
import os


import numpy as np
from numpy.testing import assert_array_equal, assert_array_almost_equal
from nose.tools import assert_raises, eq_ as eq, assert_is_none


from zarr.storage import (init_array, array_meta_key, attrs_key, DictStore, DirectoryStore,
                          ZipStore, init_group, group_meta_key, getsize, migrate_1to2, TempStore,
                          atexit_rmtree, NestedDirectoryStore, default_compressor, DBMStore)
from zarr.meta import (decode_array_metadata, encode_array_metadata, ZARR_FORMAT,
                       decode_group_metadata, encode_group_metadata)
from zarr.compat import text_type
from zarr.codecs import Zlib, Blosc, BZ2
from zarr.errors import PermissionError
from zarr.hierarchy import group


class StoreTests(object):
    """Abstract store tests."""

    def create_store(self, **kwargs):  # pragma: no cover
        # implement in sub-class
        raise NotImplementedError

    def test_get_set_del_contains(self):
        store = self.create_store()

        # test __contains__, __getitem__, __setitem__
        assert 'foo' not in store
        with assert_raises(KeyError):
            store['foo']
        store['foo'] = b'bar'
        assert 'foo' in store
        eq(b'bar', store['foo'])

        # test __delitem__ (optional)
        try:
            del store['foo']
        except NotImplementedError:
            pass
        else:
            assert 'foo' not in store
            with assert_raises(KeyError):
                store['foo']
            with assert_raises(KeyError):
                del store['foo']

    def test_writeable_values(self):
        store = self.create_store()

        # __setitem__ should accept any value that implements buffer interface
        store['foo'] = b'bar'
        store['foo'] = bytearray(b'bar')
        store['foo'] = array.array('B', b'bar')
        store['foo'] = np.frombuffer(b'bar', dtype='u1')

    def test_update(self):
        store = self.create_store()
        assert 'foo' not in store
        assert 'baz' not in store
        store.update(foo=b'bar', baz=b'quux')
        eq(b'bar', store['foo'])
        eq(b'quux', store['baz'])

    def test_iterators(self):
        store = self.create_store()

        # test iterator methods on empty store
        eq(0, len(store))
        eq(set(), set(store))
        eq(set(), set(store.keys()))
        eq(set(), set(store.values()))
        eq(set(), set(store.items()))

        # setup some values
        store['a'] = b'aaa'
        store['b'] = b'bbb'
        store['c/d'] = b'ddd'
        store['c/e/f'] = b'fff'

        # test iterators on store with data
        eq(4, len(store))
        eq({'a', 'b', 'c/d', 'c/e/f'}, set(store))
        eq({'a', 'b', 'c/d', 'c/e/f'}, set(store.keys()))
        eq({b'aaa', b'bbb', b'ddd', b'fff'}, set(store.values()))
        eq({('a', b'aaa'), ('b', b'bbb'), ('c/d', b'ddd'), ('c/e/f', b'fff')},
           set(store.items()))

    def test_pickle(self):
        store = self.create_store()
        store['foo'] = b'bar'
        store['baz'] = b'quux'
        if hasattr(store, 'flush'):
            store.flush()
        store2 = pickle.loads(pickle.dumps(store))
        eq(len(store), len(store2))
        eq(sorted(store.keys()), sorted(store2.keys()))
        eq(b'bar', store2['foo'])
        eq(b'quux', store2['baz'])

    def test_getsize(self):
        store = self.create_store()
        if hasattr(store, 'getsize'):
            eq(0, store.getsize())
            store['foo'] = b'x'
            eq(1, store.getsize())
            eq(1, store.getsize('foo'))
            store['bar'] = b'yy'
            eq(3, store.getsize())
            eq(2, store.getsize('bar'))
            store['baz'] = bytearray(b'zzz')
            eq(6, store.getsize())
            eq(3, store.getsize('baz'))
            store['quux'] = array.array('B', b'zzzz')
            eq(10, store.getsize())
            eq(4, store.getsize('quux'))
            store['spong'] = np.frombuffer(b'zzzzz', dtype='u1')
            eq(15, store.getsize())
            eq(5, store.getsize('spong'))

    def test_hierarchy(self):
        # setup
        store = self.create_store()
        store['a'] = b'aaa'
        store['b'] = b'bbb'
        store['c/d'] = b'ddd'
        store['c/e/f'] = b'fff'
        store['c/e/g'] = b'ggg'

        # check keys
        assert 'a' in store
        assert 'b' in store
        assert 'c/d' in store
        assert 'c/e/f' in store
        assert 'c/e/g' in store
        assert 'c' not in store
        assert 'c/' not in store
        assert 'c/e' not in store
        assert 'c/e/' not in store
        assert 'c/d/x' not in store

        # check __getitem__
        with assert_raises(KeyError):
            store['c']
        with assert_raises(KeyError):
            store['c/e']
        with assert_raises(KeyError):
            store['c/d/x']

        # test getsize (optional)
        if hasattr(store, 'getsize'):
            eq(6, store.getsize())
            eq(3, store.getsize('a'))
            eq(3, store.getsize('b'))
            eq(3, store.getsize('c'))
            eq(3, store.getsize('c/d'))
            eq(6, store.getsize('c/e'))
            eq(3, store.getsize('c/e/f'))
            eq(3, store.getsize('c/e/g'))
            with assert_raises(ValueError):
                store.getsize('x')
            with assert_raises(ValueError):
                store.getsize('a/x')
            with assert_raises(ValueError):
                store.getsize('c/x')
            with assert_raises(ValueError):
                store.getsize('c/x/y')
            with assert_raises(ValueError):
                store.getsize('c/d/y')
            with assert_raises(ValueError):
                store.getsize('c/d/y/z')

        # test listdir (optional)
        if hasattr(store, 'listdir'):
            eq({'a', 'b', 'c'}, set(store.listdir()))
            eq({'d', 'e'}, set(store.listdir('c')))
            eq({'f', 'g'}, set(store.listdir('c/e')))
            # no exception raised if path does not exist or is leaf
            eq([], store.listdir('x'))
            eq([], store.listdir('a/x'))
            eq([], store.listdir('c/x'))
            eq([], store.listdir('c/x/y'))
            eq([], store.listdir('c/d/y'))
            eq([], store.listdir('c/d/y/z'))
            eq([], store.listdir('c/e/f'))

        # test rmdir (optional)
        if hasattr(store, 'rmdir'):
            store.rmdir('c/e')
            assert 'c/d' in store
            assert 'c/e/f' not in store
            assert 'c/e/g' not in store
            store.rmdir('c')
            assert 'c/d' not in store
            store.rmdir()
            assert 'a' not in store
            assert 'b' not in store
            store['a'] = b'aaa'<commit_msg>Add some tests for `rename` on relevant `Store`s<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, print_function, division
import unittest
import tempfile
import atexit
import pickle
import json
import array
import shutil
import os


import numpy as np
from numpy.testing import assert_array_equal, assert_array_almost_equal
from nose.tools import assert_raises, eq_ as eq, assert_is_none


from zarr.storage import (init_array, array_meta_key, attrs_key, DictStore, DirectoryStore,
                          ZipStore, init_group, group_meta_key, getsize, migrate_1to2, TempStore,
                          atexit_rmtree, NestedDirectoryStore, default_compressor, DBMStore)
from zarr.meta import (decode_array_metadata, encode_array_metadata, ZARR_FORMAT,
                       decode_group_metadata, encode_group_metadata)
from zarr.compat import text_type
from zarr.codecs import Zlib, Blosc, BZ2
from zarr.errors import PermissionError
from zarr.hierarchy import group


class StoreTests(object):
    """Abstract store tests."""

    def create_store(self, **kwargs):  # pragma: no cover
        # implement in sub-class
        raise NotImplementedError

    def test_get_set_del_contains(self):
        store = self.create_store()

        # test __contains__, __getitem__, __setitem__
        assert 'foo' not in store
        with assert_raises(KeyError):
            store['foo']
        store['foo'] = b'bar'
        assert 'foo' in store
        eq(b'bar', store['foo'])

        # test __delitem__ (optional)
        try:
            del store['foo']
        except NotImplementedError:
            pass
        else:
            assert 'foo' not in store
            with assert_raises(KeyError):
                store['foo']
            with assert_raises(KeyError):
                del store['foo']

    def test_writeable_values(self):
        store = self.create_store()

        # __setitem__ should accept any value that implements buffer interface
        store['foo'] = b'bar'
        store['foo'] = bytearray(b'bar')
        store['foo'] = array.array('B', b'bar')
        store['foo'] = np.frombuffer(b'bar', dtype='u1')

    def test_update(self):
        store = self.create_store()
        assert 'foo' not in store
        assert 'baz' not in store
        store.update(foo=b'bar', baz=b'quux')
        eq(b'bar', store['foo'])
        eq(b'quux', store['baz'])

    def test_iterators(self):
        store = self.create_store()

        # test iterator methods on empty store
        eq(0, len(store))
        eq(set(), set(store))
        eq(set(), set(store.keys()))
        eq(set(), set(store.values()))
        eq(set(), set(store.items()))

        # setup some values
        store['a'] = b'aaa'
        store['b'] = b'bbb'
        store['c/d'] = b'ddd'
        store['c/e/f'] = b'fff'

        # test iterators on store with data
        eq(4, len(store))
        eq({'a', 'b', 'c/d', 'c/e/f'}, set(store))
        eq({'a', 'b', 'c/d', 'c/e/f'}, set(store.keys()))
        eq({b'aaa', b'bbb', b'ddd', b'fff'}, set(store.values()))
        eq({('a', b'aaa'), ('b', b'bbb'), ('c/d', b'ddd'), ('c/e/f', b'fff')},
           set(store.items()))

    def test_pickle(self):
        store = self.create_store()
        store['foo'] = b'bar'
        store['baz'] = b'quux'
        if hasattr(store, 'flush'):
            store.flush()
        store2 = pickle.loads(pickle.dumps(store))
        eq(len(store), len(store2))
        eq(sorted(store.keys()), sorted(store2.keys()))
        eq(b'bar', store2['foo'])
        eq(b'quux', store2['baz'])

    def test_getsize(self):
        store = self.create_store()
        if hasattr(store, 'getsize'):
            eq(0, store.getsize())
            store['foo'] = b'x'
            eq(1, store.getsize())
            eq(1, store.getsize('foo'))
            store['bar'] = b'yy'
            eq(3, store.getsize())
            eq(2, store.getsize('bar'))
            store['baz'] = bytearray(b'zzz')
            eq(6, store.getsize())
            eq(3, store.getsize('baz'))
            store['quux'] = array.array('B', b'zzzz')
            eq(10, store.getsize())
            eq(4, store.getsize('quux'))
            store['spong'] = np.frombuffer(b'zzzzz', dtype='u1')
            eq(15, store.getsize())
            eq(5, store.getsize('spong'))

    def test_hierarchy(self):
        # setup
        store = self.create_store()
        store['a'] = b'aaa'
        store['b'] = b'bbb'
        store['c/d'] = b'ddd'
        store['c/e/f'] = b'fff'
        store['c/e/g'] = b'ggg'

        # check keys
        assert 'a' in store
        assert 'b' in store
        assert 'c/d' in store
        assert 'c/e/f' in store
        assert 'c/e/g' in store
        assert 'c' not in store
        assert 'c/' not in store
        assert 'c/e' not in store
        assert 'c/e/' not in store
        assert 'c/d/x' not in store

        # check __getitem__
        with assert_raises(KeyError):
            store['c']
        with assert_raises(KeyError):
            store['c/e']
        with assert_raises(KeyError):
            store['c/d/x']

        # test getsize (optional)
        if hasattr(store, 'getsize'):
            eq(6, store.getsize())
            eq(3, store.getsize('a'))
            eq(3, store.getsize('b'))
            eq(3, store.getsize('c'))
            eq(3, store.getsize('c/d'))
            eq(6, store.getsize('c/e'))
            eq(3, store.getsize('c/e/f'))
            eq(3, store.getsize('c/e/g'))
            with assert_raises(ValueError):
                store.getsize('x')
            with assert_raises(ValueError):
                store.getsize('a/x')
            with assert_raises(ValueError):
                store.getsize('c/x')
            with assert_raises(ValueError):
                store.getsize('c/x/y')
            with assert_raises(ValueError):
                store.getsize('c/d/y')
            with assert_raises(ValueError):
                store.getsize('c/d/y/z')

        # test listdir (optional)
        if hasattr(store, 'listdir'):
            eq({'a', 'b', 'c'}, set(store.listdir()))
            eq({'d', 'e'}, set(store.listdir('c')))
            eq({'f', 'g'}, set(store.listdir('c/e')))
            # no exception raised if path does not exist or is leaf
            eq([], store.listdir('x'))
            eq([], store.listdir('a/x'))
            eq([], store.listdir('c/x'))
            eq([], store.listdir('c/x/y'))
            eq([], store.listdir('c/d/y'))
            eq([], store.listdir('c/d/y/z'))
            eq([], store.listdir('c/e/f'))

        # test rename (optional)
        if hasattr(store, 'rename'):
            store.rename('c/e', 'c/e2')
            assert 'c/d' in store
            assert 'c/e' not in store
            assert 'c/e/f' not in store
            assert 'c/e/g' not in store
            assert 'c/e2' not in store
            assert 'c/e2/f' in store
            assert 'c/e2/g' in store
            store.rename('c/e2', 'c/e')
            assert 'c/d' in store
            assert 'c/e2' not in store
            assert 'c/e2/f' not in store
            assert 'c/e2/g' not in store
            assert 'c/e' not in store
            assert 'c/e/f' in store
            assert 'c/e/g' in store
            store.rename('c', 'c1/c2/c3')
            assert 'a' in store
            assert 'c' not in store
            assert 'c/d' not in store
            assert 'c/e' not in store
            assert 'c/e/f' not in store
            assert 'c/e/g' not in store
            assert 'c1' not in store
            assert 'c1/c2' not in store
            assert 'c1/c2/c3' not in store
            assert 'c1/c2/c3/d' in store
            assert 'c1/c2/c3/e' not in store
            assert 'c1/c2/c3/e/f' in store
            assert 'c1/c2/c3/e/g' in store
            store.rename('c1/c2/c3', 'c')
            assert 'c' not in store
            assert 'c/d' in store
            assert 'c/e' not in store
            assert 'c/e/f' in store
            assert 'c/e/g' in store
            assert 'c1' not in store
            assert 'c1/c2' not in store
            assert 'c1/c2/c3' not in store
            assert 'c1/c2/c3/d' not in store
            assert 'c1/c2/c3/e' not in store
            assert 'c1/c2/c3/e/f' not in store
            assert 'c1/c2/c3/e/g' not in store

        # test rmdir (optional)
        if hasattr(store, 'rmdir'):
            store.rmdir('c/e')
            assert 'c/d' in store
            assert 'c/e/f' not in store
            assert 'c/e/g' not in store
            store.rmdir('c')
            assert 'c/d' not in store
            store.rmdir()
            assert 'a' not in store
            assert 'b' not in store
            store['a'] = b'aaa'
{'fraction_non_alphanumeric': 0.14753579120739896, 'fraction_numerical': 0.007474977828455594, 'mean_word_length': 1.848610609888127, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1177490', 'n_tokens_mistral': 5168, 'n_tokens_neox': 5063, 'n_words': 1224}