<commit_before>from storage import *
import pytest

# NOTE: you can add other SQL backends by subclassing and changing setup
class TestSQLStorageBackend(object):
    def setup(self):
        self._delete_tmp_files()
        self.storage = SQLStorageBackend(":memory:")
        self.schema = {
            'uuid': [('uuid', 'uuid'),
                     ('table', 'int'),
                     ('row', 'int')],
            'samples': [('replica', 'int'),
                        ('ensemble', 'uuid'),
                        ('trajectory', 'uuid')]
        }
        self.storage.register_schema(self.schema)

    def teardown(self):
        self._delete_tmp_files()

    @staticmethod
    def _delete_tmp_files():
        if os.path.isfile("test.sql"):
            os.remove("test.sql")

    def test_extract_metadata(self):
        pass

    def test_filename_from_backend(self):
        pass

    def _col_names_set(self, table):
        meta = self.storage.metadata
        return set([col.name for col in meta.tables[table].columns])

    def test_setup(self):
        table_names = self.storage.engine.table_names()
        assert set(table_names) == set(['uuid', 'samples'])
        assert self._col_names_set('uuid') == set(['uuid', 'table', 'row'])
        assert self._col_names_set('samples') == set(['replica', 'ensemble',
                                                      'trajectory'])

    def test_register_schema(self):
        new_schema = {
            'snapshot0': [('filename', 'str'),
                          ('index', 'int')]
        }
        self.storage.register_schema(new_schema)
        table_names = self.storage.engine.table_names()
        assert set(table_names) == set(['uuid', 'samples', 'snapshot0'])
        assert self._col_names_set('snapshot0') == set(['filename',
                                                        'index'])

    def test_register_schema_modify_fails(self):
        # run register_schema on self.schema again
        pass

    def test_add_to_table(self):
        pass

    def test_load_table_data(self):
        pass

    def test_persistence(self):
        pass<commit_msg>more tests for sql-based storage<commit_after>from storage import *
import pytest

class TestSQLStorageBackend(object):
    def setup(self):
        self._delete_tmp_files()
        self.storage = self._default_backend
        self.schema = {
            'uuid': [('uuid', 'uuid'),
                     ('table', 'int'),
                     ('row', 'int')],
            'samples': [('replica', 'int'),
                        ('ensemble', 'uuid'),
                        ('trajectory', 'uuid')]
        }
        self.storage.register_schema(self.schema)

    def teardown(self):
        self._delete_tmp_files()

    @property
    def _default_backend(self):
        # NOTE: test other SQL backends by subclassing and changing this
        return SQLStorageBackend(":memory:")

    @staticmethod
    def _delete_tmp_files():
        if os.path.isfile("test.sql"):
            os.remove("test.sql")

    def test_extract_metadata(self):
        pytest.skip()

    @pytest.mark.parametrize('test_input,expected', [
        (("file.sql", "sqlite"), "sqlite:///file.sql"),
        ((":memory:", "sqlite"), "sqlite:///:memory:")
    ])
    def test_filename_from_backend(self, test_input, expected):
        filename_from_backend = self.storage.filename_from_backend
        assert filename_from_backend(*test_input) == expected

    def test_filename_from_backend_unknown(self):
        with pytest.raises(KeyError):
            self.storage.filename_from_backend("file.sql", "foo")

    def _col_names_set(self, table):
        meta = self.storage.metadata
        return set([col.name for col in meta.tables[table].columns])

    def test_setup(self):
        table_names = self.storage.engine.table_names()
        assert set(table_names) == set(['uuid', 'samples'])
        assert self._col_names_set('uuid') == set(['uuid', 'table', 'row'])
        assert self._col_names_set('samples') == set(['replica', 'ensemble',
                                                      'trajectory'])

    def test_register_schema(self):
        new_schema = {
            'snapshot0': [('filename', 'str'),
                          ('index', 'int')]
        }
        self.storage.register_schema(new_schema)
        table_names = self.storage.engine.table_names()
        assert set(table_names) == set(['uuid', 'samples', 'snapshot0'])
        assert self._col_names_set('snapshot0') == set(['filename',
                                                        'index'])

    def test_register_schema_modify_fails(self):
        with pytest.raises(TypeError):
            self.storage.register_schema(self.schema)

    def test_add_to_table(self):
        pytest.skip()

    def test_load_table_data(self):
        pytest.skip()

    def test_persistence(self):
        pytest.skip()
{'fraction_non_alphanumeric': 0.13336043911364098, 'fraction_numerical': 0.0012197601138442774, 'mean_word_length': 2.1457800511508953, 'pattern_counts': {'":': 3, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3851889', 'n_tokens_mistral': 1395, 'n_tokens_neox': 1384, 'n_words': 264}