<commit_before>import unittest
from openquake.nrmllib.readers import Reader, StringReader, collect_readers


class ReaderTestCase(unittest.TestCase):
    fake = StringReader('fake', '<gmfSet/>', '''\
lon,lat,gmv
1.0,2.0,0.1
1.0,2.1,0.1
1.0,2.2,0.2
1.0,2.3,0.2
''')

    def test_getitem(self):
        # test that a Reader object support the bracket notation
        self.assertEqual(self.fake[0],
                         {'lon': '1.0', 'lat': '2.0', 'gmv': '0.1'})
        self.assertEqual(self.fake[1],
                         {'lon': '1.0', 'lat': '2.1', 'gmv': '0.1'})
        self.assertEqual(
            self.fake[2:4],
            [{'lon': '1.0', 'lat': '2.2', 'gmv': '0.2'},
             {'lon': '1.0', 'lat': '2.3', 'gmv': '0.2'}])

    def test_collect_readers(self):
        # test the logic of the reader generator
        fnames = ['a.csv', 'a.mdata',  # group of one pair
                  'x__0.mdata', 'x__0.csv',  # group of two pairs
                  'x__1.mdata', 'x__1.csv',  # group of two pairs
                  'y.csv',  # unpaired file
                  'x__2.mdata',  # another unpaired file
                  'x__2.cs',  # ignored extension
                  ]
        got = []
        for name, readers in collect_readers(Reader, None, fnames):
            got.append((name, ' '.join(reader.name for reader in readers)))
        self.assertEqual(got, [('a', 'a'), ('x', 'x__0 x__1')])<commit_msg>Improved a comment<commit_after>import unittest
from openquake.nrmllib.readers import Reader, StringReader, collect_readers


class ReaderTestCase(unittest.TestCase):
    fake = StringReader('fake', '<gmfSet/>', '''\
lon,lat,gmv
1.0,2.0,0.1
1.0,2.1,0.1
1.0,2.2,0.2
1.0,2.3,0.2
''')

    def test_getitem(self):
        # test that a Reader object support the bracket notation
        self.assertEqual(self.fake[0],
                         {'lon': '1.0', 'lat': '2.0', 'gmv': '0.1'})
        self.assertEqual(self.fake[1],
                         {'lon': '1.0', 'lat': '2.1', 'gmv': '0.1'})
        self.assertEqual(
            self.fake[2:4],
            [{'lon': '1.0', 'lat': '2.2', 'gmv': '0.2'},
             {'lon': '1.0', 'lat': '2.3', 'gmv': '0.2'}])

    def test_collect_readers(self):
        # test the logic of the reader generator, in particular the
        # `<groupname>__` convention in the name of the files to specify
        # files belonging to the same group
        # also test the lexicographic ordering of the readers
        fnames = ['a.csv', 'a.mdata',  # group of one pair
                  'x__1.mdata', 'x__1.csv', 'x__0.mdata', 'x__0.csv',  # group
                  'y.csv',  # unpaired file
                  'x__2.mdata',  # another unpaired file
                  'x__2.cs',  # ignored extension
                  ]
        got = []
        for groupname, readers in collect_readers(Reader, None, fnames):
            got.append((groupname, ' '.join(r.name for r in readers)))
        self.assertEqual(got, [('a', 'a'), ('x', 'x__0 x__1')])
{'fraction_non_alphanumeric': 0.17531603459747172, 'fraction_numerical': 0.03992015968063872, 'mean_word_length': 2.0435222672064777, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5769577', 'n_tokens_mistral': 1072, 'n_tokens_neox': 1033, 'n_words': 251}