<commit_before>from unittest import TestCase

from pantsmud.driver import command, parser


class TestParse(TestCase):
    def setUp(self):
        self.parser = parser.Parser()

    def test_parse_works_for_one_word(self):
        pattern = [("foo", parser.WORD)]
        token_string = "bar"
        expected = {
            "foo": "bar"
        }
        self.assertDictEqual(expected, self.parser.parse(pattern, token_string))

    def test_parse_works_for_one_int(self):
        pattern = [("foo", parser.INT)]
        token_string = "123"
        expected = {
            "foo": 123
        }
        self.assertDictEqual(expected, self.parser.parse(pattern, token_string))

    def test_parse_works_for_one_string(self):
        pattern = [("foo", parser.STRING)]
        token_string = "foo bar 123 hello world"
        expected = {
            "foo": "foo bar 123 hello world"
        }
        self.assertDictEqual(expected, self.parser.parse(pattern, token_string))

    def test_parse_works_for_word_int_and_string(self):
        pattern = [("foo", parser.WORD), ("bar", parser.INT), ("baz", parser.STRING)]
        token_string = "abc 123 hello 987 world"
        expected = {
            "foo": "abc",
            "bar": 123,
            "baz": "hello 987 world"
        }
        self.assertDictEqual(expected, self.parser.parse(pattern, token_string))

    def test_parse_words_for_word_int_and_string_with_variable_length_whitespace(self):
        pattern = [("foo", parser.WORD), ("bar", parser.INT), ("baz", parser.STRING)]
        token_string = "abc   123  hello    987  world"
        expected = {
            "foo": "abc",
            "bar": 123,
            "baz": "hello    987  world"
        }
        self.assertDictEqual(expected, self.parser.parse(pattern, token_string))

    def test_parse_works_with_leading_whitespace(self):
        pattern = [("foo", parser.WORD)]
        token_string = "   bar"
        expected = {
            "foo": "bar"
        }
        self.assertDictEqual(expected, self.parser.parse(pattern, token_string))

    def test_parse_raises_PatternError_if_token_type_does_not_exist(self):
        pattern = [("foo", "bar")]
        token_string = "baz"
        self.assertRaises(parser.PatternError, self.parser.parse, pattern, token_string)

    def test_parse_raises_CommandError_if_int_param_is_not_an_int(self):
        pattern = [("foo", parser.INT)]
        token_string = "bar"
        self.assertRaises(command.CommandError, self.parser.parse, pattern, token_string)

    def test_parse_raises_CommandError_if_pattern_has_rules_but_token_string_is_only_whitespace(self):
        pattern = [("foo", parser.WORD)]
        token_string = "    "
        self.assertRaises(command.CommandError, self.parser.parse, pattern, token_string)

    def test_parse_raises_CommandError_if_number_of_params_is_less_than_length_of_pattern(self):
        pattern = [("foo", parser.WORD), ("bar", parser.WORD)]
        token_string = "baz"
        self.assertRaises(command.CommandError, self.parser.parse, pattern, token_string)

    def test_parse_raises_CommandError_if_number_of_params_is_greater_than_length_of_pattern(self):
        pattern = [("foo", parser.WORD)]
        token_string = "bar baz"
        self.assertRaises(command.CommandError, self.parser.parse, pattern, token_string)


class TestAddTokenType(TestCase):
    def setUp(self):
        self.parser = parser.Parser()

    def test_add_token_type_raises_TypeError_when_type_func_is_not_callable(self):
        self.assertRaises(TypeError, self.parser.add_token_type, "test", None)
<commit_msg>Fixed test failures in test_parser.py caused by an earlier commit which moved exceptions to a different module.<commit_after>from unittest import TestCase

from pantsmud.driver import command, error, parser


class TestParse(TestCase):
    def setUp(self):
        self.parser = parser.Parser()

    def test_parse_works_for_one_word(self):
        pattern = [("foo", parser.WORD)]
        token_string = "bar"
        expected = {
            "foo": "bar"
        }
        self.assertDictEqual(expected, self.parser.parse(pattern, token_string))

    def test_parse_works_for_one_int(self):
        pattern = [("foo", parser.INT)]
        token_string = "123"
        expected = {
            "foo": 123
        }
        self.assertDictEqual(expected, self.parser.parse(pattern, token_string))

    def test_parse_works_for_one_string(self):
        pattern = [("foo", parser.STRING)]
        token_string = "foo bar 123 hello world"
        expected = {
            "foo": "foo bar 123 hello world"
        }
        self.assertDictEqual(expected, self.parser.parse(pattern, token_string))

    def test_parse_works_for_word_int_and_string(self):
        pattern = [("foo", parser.WORD), ("bar", parser.INT), ("baz", parser.STRING)]
        token_string = "abc 123 hello 987 world"
        expected = {
            "foo": "abc",
            "bar": 123,
            "baz": "hello 987 world"
        }
        self.assertDictEqual(expected, self.parser.parse(pattern, token_string))

    def test_parse_words_for_word_int_and_string_with_variable_length_whitespace(self):
        pattern = [("foo", parser.WORD), ("bar", parser.INT), ("baz", parser.STRING)]
        token_string = "abc   123  hello    987  world"
        expected = {
            "foo": "abc",
            "bar": 123,
            "baz": "hello    987  world"
        }
        self.assertDictEqual(expected, self.parser.parse(pattern, token_string))

    def test_parse_works_with_leading_whitespace(self):
        pattern = [("foo", parser.WORD)]
        token_string = "   bar"
        expected = {
            "foo": "bar"
        }
        self.assertDictEqual(expected, self.parser.parse(pattern, token_string))

    def test_parse_raises_PatternError_if_token_type_does_not_exist(self):
        pattern = [("foo", "bar")]
        token_string = "baz"
        self.assertRaises(error.PatternError, self.parser.parse, pattern, token_string)

    def test_parse_raises_CommandError_if_int_param_is_not_an_int(self):
        pattern = [("foo", parser.INT)]
        token_string = "bar"
        self.assertRaises(error.CommandError, self.parser.parse, pattern, token_string)

    def test_parse_raises_CommandError_if_pattern_has_rules_but_token_string_is_only_whitespace(self):
        pattern = [("foo", parser.WORD)]
        token_string = "    "
        self.assertRaises(error.CommandError, self.parser.parse, pattern, token_string)

    def test_parse_raises_CommandError_if_number_of_params_is_less_than_length_of_pattern(self):
        pattern = [("foo", parser.WORD), ("bar", parser.WORD)]
        token_string = "baz"
        self.assertRaises(error.CommandError, self.parser.parse, pattern, token_string)

    def test_parse_raises_CommandError_if_number_of_params_is_greater_than_length_of_pattern(self):
        pattern = [("foo", parser.WORD)]
        token_string = "bar baz"
        self.assertRaises(error.CommandError, self.parser.parse, pattern, token_string)


class TestAddTokenType(TestCase):
    def setUp(self):
        self.parser = parser.Parser()

    def test_add_token_type_raises_TypeError_when_type_func_is_not_callable(self):
        self.assertRaises(TypeError, self.parser.add_token_type, "test", None)

{'fraction_non_alphanumeric': 0.11136922656463515, 'fraction_numerical': 0.009838753757857338, 'mean_word_length': 3.0347298787210586, 'pattern_counts': {'":': 20, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3813585', 'n_tokens_mistral': 2335, 'n_tokens_neox': 2222, 'n_words': 413}