<commit_before>import tempfile
import unittest

from fito.data_store.file import FileDataStore, RawSerializer, PickleSerializer
from test_data_store import delete
from test_spec import get_test_specs


class TestFileDataStore(unittest.TestCase):
    def setUp(self):
        self.ds = FileDataStore(tempfile.mktemp(), serializer=RawSerializer())

        self.test_specs = get_test_specs(only_lists=True)

    def tearDown(self):
        delete(self.ds.path)

    def test_existing_path(self):
        assert self.ds.serializer == FileDataStore(self.ds.path).serializer
        self.assertRaises(RuntimeError, FileDataStore, self.ds.path, serializer=PickleSerializer())

    def test_clean(self):
        self.ds[self.test_specs[0]] = ""

        try:
            self.ds.iterkeys().next()
        except:
            assert False

        self.ds.clean()

        self.assertRaises(StopIteration, self.ds.iterkeys().next)




<commit_msg>More tests for FileDataStore<commit_after>import tempfile
import unittest

from fito.data_store.file import FileDataStore, RawSerializer, PickleSerializer
from test_data_store import delete
from test_spec import get_test_specs


class TestFileDataStore(unittest.TestCase):
    def setUp(self):
        self.data_stores = [
            FileDataStore(tempfile.mktemp(), serializer=RawSerializer()),
            FileDataStore(tempfile.mktemp(), use_class_name=True),
            FileDataStore(tempfile.mktemp(), serializer=PickleSerializer()),
        ]

        self.test_specs = get_test_specs(only_lists=True)

    def tearDown(self):
        for ds in self.data_stores:
            delete(ds.path)

    def test_existing_path(self):
        for ds in self.data_stores:
            self.assertRaises(RuntimeError, FileDataStore, ds.path, use_class_name=not ds.use_class_name)
            other_serializer = PickleSerializer() if isinstance(ds.serializer, RawSerializer) else RawSerializer()
            self.assertRaises(RuntimeError, FileDataStore, ds.path, serializer=other_serializer)

    def test_clean(self):
        for ds in self.data_stores:
            ds[self.test_specs[0]] = ""

            try:
                ds.iterkeys().next()
            except:
                assert False

            ds.clean()

            self.assertRaises(StopIteration, ds.iterkeys().next)





{'fraction_non_alphanumeric': 0.09278350515463918, 'fraction_numerical': 0.000859106529209622, 'mean_word_length': 3.008605851979346, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6862110', 'n_tokens_mistral': 711, 'n_tokens_neox': 679, 'n_words': 128}