<commit_before>import os
import unittest

from storages import metadata

class SchemeTest(unittest.TestCase):

    def test_create_scheme(self):
        
        scheme = metadata.Scheme()
        self.assertEqual(type(scheme), metadata.Scheme)


    def test_scheme_as_dict(self):

        scheme = metadata.Scheme()
        self.assertEqual(scheme.as_dict(), {'item_id': None, 'key': None, 'value': None})        


class MetadataTest(unittest.TestCase):

    def test_create_metadata(self):

        database_path = ':memory:'
        meta = metadata.Metadata(database_path)
        self.assertTrue(os.path.exists(database_path))

    def test_store_metadata(self):

        database_path = ':memory:'
        meta = metadata.Metadata(database_path)
        self.assertTrue(os.path.exists(database_path))

        self.assertRaises(RuntimeError, meta.put, None, None)
        self.assertRaises(RuntimeError, meta.put, 'a1', None)
        meta.put('b1', [('k1','v1'),])
        meta.put('b1', [('k1','v1'),('k2','v2')])
        meta.put('b1', [('k2','v3'),('k3','v2')])
        self.assertEqual(meta.get('b1'), ('b1', [('k1','v1'),('k2','v2'),('k2','v3'),('k3','v2')]))

        meta.delete('b1')
        self.assertEqual(meta.get('b1'), ('b1', []))


    def test_get_metadata(self):

        database_path = ':memory:'
        meta = metadata.Metadata(database_path)
        self.assertTrue(os.path.exists(database_path))

        meta.put('b1', [('k1','v1')])
        meta.put('b2', [('k2','v2')])
        meta.put('b3', [('k3','v3')])
        self.assertEqual(meta.get(), [(u'b1', [(u'k1', u'v1')]), (u'b2', [(u'k2', u'v2')]), (u'b3', [(u'k3', u'v3')])])

        meta.delete(['b1', 'b2', 'b3'])
        self.assertEqual(meta.count(),0)
        

    def test_search_metadata(self):

        database_path = ':memory:'
        meta = metadata.Metadata(database_path)
        self.assertTrue(os.path.exists(database_path))

        meta.put('b1', [('k1','v1')])
        meta.put('b2', [('k1','v2')])
        meta.put('b3', [('k1','v3')])

        self.assertEqual(meta.search('k1', 'v2'), ['b2'])
        

<commit_msg>[+] remove old references to file based db<commit_after>import os
import unittest

from storages import metadata

class SchemeTest(unittest.TestCase):

    def test_create_scheme(self):
        
        scheme = metadata.Scheme()
        self.assertEqual(type(scheme), metadata.Scheme)


    def test_scheme_as_dict(self):

        scheme = metadata.Scheme()
        self.assertEqual(scheme.as_dict(), {'item_id': None, 'key': None, 'value': None})        


class MetadataTest(unittest.TestCase):

    def test_create_metadata(self):

        database_path = ':memory:'
        meta = metadata.Metadata(database_path)
        self.assertTrue(type(meta), metadata.Metadata)

    def test_store_metadata(self):

        database_path = ':memory:'
        meta = metadata.Metadata(database_path)

        self.assertRaises(RuntimeError, meta.put, None, None)
        self.assertRaises(RuntimeError, meta.put, 'a1', None)
        meta.put('b1', [('k1','v1'),])
        meta.put('b1', [('k1','v1'),('k2','v2')])
        meta.put('b1', [('k2','v3'),('k3','v2')])
        self.assertEqual(meta.get('b1'), ('b1', [('k1','v1'),('k2','v2'),('k2','v3'),('k3','v2')]))

        meta.delete('b1')
        self.assertEqual(meta.get('b1'), ('b1', []))


    def test_get_metadata(self):

        database_path = ':memory:'
        meta = metadata.Metadata(database_path)

        meta.put('b1', [('k1','v1')])
        meta.put('b2', [('k2','v2')])
        meta.put('b3', [('k3','v3')])
        self.assertEqual(meta.get(), [(u'b1', [(u'k1', u'v1')]), (u'b2', [(u'k2', u'v2')]), (u'b3', [(u'k3', u'v3')])])

        meta.delete(['b1', 'b2', 'b3'])
        self.assertEqual(meta.count(),0)
        

    def test_search_metadata(self):

        database_path = ':memory:'
        meta = metadata.Metadata(database_path)

        meta.put('b1', [('k1','v1')])
        meta.put('b2', [('k1','v2')])
        meta.put('b3', [('k1','v3')])

        self.assertEqual(meta.search('k1', 'v2'), ['b2'])
        


{'fraction_non_alphanumeric': 0.21595330739299612, 'fraction_numerical': 0.029669260700389104, 'mean_word_length': 3.0562130177514795, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6500977', 'n_tokens_mistral': 1486, 'n_tokens_neox': 1430, 'n_words': 217}