<commit_before>import unittest

from saw.parser.blocks import Blocks

class Test_Saw(unittest.TestCase):

    def setUp(self):
        pass

    def test_blocks(self):
        res = Blocks.parse('John, stand up!')
        self.assertEqual(res, [['John', ','], ['stand up!', '']])

        res = Blocks.parse('Enter that: pin code.')
        self.assertEqual(res, [['Enter that', ':'], ['pin code.', '']])

        res = Blocks.parse('Limite it:: hey,')
        self.assertEqual(res, [['Limite it', '::'], ['hey', ',']])

if __name__ == "__main__":
    unittest.main()<commit_msg>Added rules for parsers tests<commit_after>import unittest

from saw.parser.blocks import Blocks

class Test_Saw(unittest.TestCase):

    def setUp(self):
        pass

    def test_blocks(self):
        single_rules = [',', ':', '=', '+', ';', '*']
        for rule in single_rules:
            """
            Test with:
            space after, space before, space around, another symbols
            few spaces - to one space
            """
            t_rule = tuple([rule])
            text = "This example%s  with %s anoher. comb!  of %sspaces" % [rule] * 3
            expect = ['This example', t_rule, 'with', t_rule, 'another. comb!  of', t_rule, 'spaces']
            self.assertEqual(Blocks.parse(text), expect)

            """
            Test with:
            dublicated symbols, another symbol beetwen dublicated (or from this range)
            symbol in end of string - not empty value
            """
            d_rule = tuple([rule, rule])
            d_with_mixin = tuple([rule, ',', rule])
            text = "This example%s%s with %s,%s anoher %s.%s spaces%s" % [rule] * 7
            expect = ['This example', d_rule, 'with', d_with_mixin, 'another',  t_rule, '.', t_rule, 'spaces', t_rule]
            self.assertEqual(Blocks.parse(text), expect)

            #@TODO: "-"

        double_rule_begin = ['{', '(', '[', '"', "'"]
        double_rule_end   = ['}', ')', ']', '"', "'"]
        
        double_rules = zip(double_rule_begin, double_rule_end)

        for rule in double_rules:
            



        res = Blocks.parse('Nine - number')
        self.assertEqual(res, ['Nine', ('-'), 'number'])

        res = Blocks.parse('Cat (or dog) name')
        self.assertEqual(res, ['cat', ('('), 'or dog', (')'), 'name')

        res = Blocks.parse('Cat (or dog (or poni)) name')
        self.assertEqual(res, ['cat', ('('), 'or dog', ('('), 'or poni', (')', ')'), 'name')

        res = Blocks.parse('The Car "Micro" fired')
        self.assertEqual(res, ['The Car', ('"'), 'Micro', ('"'), 'fired'])

if __name__ == "__main__":
    unittest.main()
{'fraction_non_alphanumeric': 0.18397291196388263, 'fraction_numerical': 0.0007524454477050414, 'mean_word_length': 2.32375, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3910842', 'n_tokens_mistral': 859, 'n_tokens_neox': 830, 'n_words': 208}