<commit_before>import unittest
import numpy
from openquake.commonlib.datastore import DataStore


class DataStoreTestCase(unittest.TestCase):
    def setUp(self):
        self.dstore = DataStore()

    def test_pik(self):
        # store pickleable Python objects
        self.dstore['key1'] = 'value1'
        self.assertEqual(len(self.dstore), 1)
        self.dstore['key2'] = 'value2'
        self.assertEqual(list(self.dstore), 'key1 key2'.split())
        del self.dstore['key2']
        self.assertEqual(list(self.dstore), ['key1'])
        self.assertEqual(self.dstore['key1'], 'value1')
        self.dstore.remove()

    def test_hdf5(self):
        # store numpy arrays as hdf5 files
        with self.assertRaises(ValueError):
            self.dstore['key1_h5'] = 'value1'
        self.assertEqual(len(self.dstore), 0)
        self.dstore['key1_h5'] = value1 = numpy.array(['a', 'b'])
        self.dstore['key2_h5'] = numpy.array([1, 2])
        self.assertEqual(list(self.dstore), 'key1_h5 key2_h5'.split())
        del self.dstore['key2_h5']
        self.assertEqual(list(self.dstore), ['key1_h5'])<commit_msg>Made the hdf5 test optional<commit_after>import unittest
import numpy
from openquake.commonlib.datastore import DataStore


class DataStoreTestCase(unittest.TestCase):
    def setUp(self):
        self.dstore = DataStore()

    def test_pik(self):
        # store pickleable Python objects
        self.dstore['key1'] = 'value1'
        self.assertEqual(len(self.dstore), 1)
        self.dstore['key2'] = 'value2'
        self.assertEqual(list(self.dstore), 'key1 key2'.split())
        del self.dstore['key2']
        self.assertEqual(list(self.dstore), ['key1'])
        self.assertEqual(self.dstore['key1'], 'value1')
        self.dstore.remove()

    def test_hdf5(self):
        # optional test, run only if h5py is available
        try:
            import h5py
        except ImportError:
            raise unittest.SkipTest

        # store numpy arrays as hdf5 files
        with self.assertRaises(ValueError):
            self.dstore['key1_h5'] = 'value1'
        self.assertEqual(len(self.dstore), 0)
        self.dstore['key1_h5'] = value1 = numpy.array(['a', 'b'])
        self.dstore['key2_h5'] = numpy.array([1, 2])
        self.assertEqual(list(self.dstore), 'key1_h5 key2_h5'.split())
        del self.dstore['key2_h5']
        self.assertEqual(list(self.dstore), ['key1_h5'])
{'fraction_non_alphanumeric': 0.13954451345755695, 'fraction_numerical': 0.02774327122153209, 'mean_word_length': 2.8780096308186196, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2391061', 'n_tokens_mistral': 809, 'n_tokens_neox': 794, 'n_words': 146}