<commit_before>"""
Runs IRC parser tests from ircdocs/parser-tests.

This test suite runs static code only.
"""
from pathlib import Path
import unittest

import yaml

PARSER_DATA_PATH = Path(__file__).parent.resolve() / 'parser-tests' / 'tests'
print(PARSER_DATA_PATH)

from pylinkirc import utils
from pylinkirc.protocols.ircs2s_common import IRCCommonProtocol

class MessageParserTest(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        with open(PARSER_DATA_PATH / 'msg-split.yaml') as f:
            cls.MESSAGE_SPLIT_TEST_DATA = yaml.safe_load(f)
        with open(PARSER_DATA_PATH / 'userhost-split.yaml') as f:
            cls.USER_HOST_SPLIT_TEST_DATA = yaml.safe_load(f)
        with open(PARSER_DATA_PATH / 'mask-match.yaml') as f:
            cls.MASK_MATCH_TEST_DATA = yaml.safe_load(f)

    def testMessageSplit(self):
        for testdata in self.MESSAGE_SPLIT_TEST_DATA['tests']:
            inp = testdata['input']
            atoms = testdata['atoms']

            with self.subTest():
                expected = []
                has_source = False
                if 'source' in atoms:
                    has_source = True
                    expected.append(atoms['source'])

                if 'verb' in atoms:
                    expected.append(atoms['verb'])

                if 'params' in atoms:
                    expected.extend(atoms['params'])

                if 'tags' in atoms:
                    # Remove message tags before parsing
                    _, inp = inp.split(" ", 1)

                if has_source:
                    parts = IRCCommonProtocol.parse_prefixed_args(inp)
                else:
                    parts = IRCCommonProtocol.parse_args(inp)
                self.assertEqual(expected, parts, "Parse test failed for string: %r" % inp)

    @unittest.skip("Not quite working yet")
    def testMessageTags(self):
        for testdata in self.MESSAGE_SPLIT_TEST_DATA['tests']:
            inp = testdata['input']
            atoms = testdata['atoms']

            with self.subTest():
                if 'tags' in atoms:
                    self.assertEqual(atoms['tags'], IRCCommonProtocol.parse_message_tags(inp.split(" ")),
                                     "Parse test failed for message tags: %r" % inp)

    def testUserHostSplit(self):
        for test in self.USER_HOST_SPLIT_TEST_DATA['tests']:
            inp = test['source']
            atoms = test['atoms']

            with self.subTest():
                if 'nick' not in atoms or 'user' not in atoms or 'host' not in atoms:
                    # Trying to parse a hostmask with missing atoms is an error in split_hostmask()
                    with self.assertRaises(ValueError):
                        utils.split_hostmask(inp)
                else:
                    expected = [atoms['nick'], atoms['user'], atoms['host']]
                    self.assertEqual(expected, utils.split_hostmask(inp))

    def testHostMatch(self):
        for test in self.MASK_MATCH_TEST_DATA['tests']:
            mask = test['mask']

            # N.B.: utils.match_text() does Unicode case-insensitive match by default,
            # which might not be the right thing to do on IRC.
            # But irc.to_lower() isn't a static function so we're not testing it here...
            for match in test['matches']:
                with self.subTest():
                    self.assertTrue(utils.match_text(mask, match))

            for fail in test['fails']:
                with self.subTest():
                    self.assertFalse(utils.match_text(mask, fail))

if __name__ == '__main__':
    unittest.main()
<commit_msg>Add validate-hostname tests from ircdocs/parser-tests<commit_after>"""
Runs IRC parser tests from ircdocs/parser-tests.

This test suite runs static code only.
"""
from pathlib import Path
import unittest

import yaml

PARSER_DATA_PATH = Path(__file__).parent.resolve() / 'parser-tests' / 'tests'
print(PARSER_DATA_PATH)

from pylinkirc import utils
from pylinkirc.protocols.ircs2s_common import IRCCommonProtocol

class MessageParserTest(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        with open(PARSER_DATA_PATH / 'msg-split.yaml') as f:
            cls.MESSAGE_SPLIT_TEST_DATA = yaml.safe_load(f)
        with open(PARSER_DATA_PATH / 'userhost-split.yaml') as f:
            cls.USER_HOST_SPLIT_TEST_DATA = yaml.safe_load(f)
        with open(PARSER_DATA_PATH / 'mask-match.yaml') as f:
            cls.MASK_MATCH_TEST_DATA = yaml.safe_load(f)
        with open(PARSER_DATA_PATH / 'validate-hostname.yaml') as f:
            cls.VALIDATE_HOSTNAME_TEST_DATA = yaml.safe_load(f)

    def testMessageSplit(self):
        for testdata in self.MESSAGE_SPLIT_TEST_DATA['tests']:
            inp = testdata['input']
            atoms = testdata['atoms']

            with self.subTest():
                expected = []
                has_source = False
                if 'source' in atoms:
                    has_source = True
                    expected.append(atoms['source'])

                if 'verb' in atoms:
                    expected.append(atoms['verb'])

                if 'params' in atoms:
                    expected.extend(atoms['params'])

                if 'tags' in atoms:
                    # Remove message tags before parsing
                    _, inp = inp.split(" ", 1)

                if has_source:
                    parts = IRCCommonProtocol.parse_prefixed_args(inp)
                else:
                    parts = IRCCommonProtocol.parse_args(inp)
                self.assertEqual(expected, parts, "Parse test failed for string: %r" % inp)

    @unittest.skip("Not quite working yet")
    def testMessageTags(self):
        for testdata in self.MESSAGE_SPLIT_TEST_DATA['tests']:
            inp = testdata['input']
            atoms = testdata['atoms']

            with self.subTest():
                if 'tags' in atoms:
                    self.assertEqual(atoms['tags'], IRCCommonProtocol.parse_message_tags(inp.split(" ")),
                                     "Parse test failed for message tags: %r" % inp)

    def testUserHostSplit(self):
        for test in self.USER_HOST_SPLIT_TEST_DATA['tests']:
            inp = test['source']
            atoms = test['atoms']

            with self.subTest():
                if 'nick' not in atoms or 'user' not in atoms or 'host' not in atoms:
                    # Trying to parse a hostmask with missing atoms is an error in split_hostmask()
                    with self.assertRaises(ValueError):
                        utils.split_hostmask(inp)
                else:
                    expected = [atoms['nick'], atoms['user'], atoms['host']]
                    self.assertEqual(expected, utils.split_hostmask(inp))

    def testHostMatch(self):
        for test in self.MASK_MATCH_TEST_DATA['tests']:
            mask = test['mask']

            # N.B.: utils.match_text() does Unicode case-insensitive match by default,
            # which might not be the right thing to do on IRC.
            # But irc.to_lower() isn't a static function so we're not testing it here...
            for match in test['matches']:
                with self.subTest():
                    self.assertTrue(utils.match_text(mask, match))

            for fail in test['fails']:
                with self.subTest():
                    self.assertFalse(utils.match_text(mask, fail))

    def testValidateHostname(self):
        for test in self.VALIDATE_HOSTNAME_TEST_DATA['tests']:
            with self.subTest():
                self.assertEqual(test['valid'], IRCCommonProtocol.is_server_name(test['host']),
                                 "Failed test for %r; should be %s" % (test['host'], test['valid']))


    # N.B. skipping msg-join tests because PyLink doesn't think about messages that way

if __name__ == '__main__':
    unittest.main()

{'fraction_non_alphanumeric': 0.09975976735364775, 'fraction_numerical': 0.0005057529396889619, 'mean_word_length': 1.9296296296296296, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5076580', 'n_tokens_mistral': 2288, 'n_tokens_neox': 2176, 'n_words': 562}