<commit_before>import unittest

from kissup import lexer


def lex(s):
    return list(lexer.lex_kissup(s))


class TestLexer(unittest.TestCase):
    def test_single_tokens_outside_brackets(self):
        self.assertEqual(lex('['), [
            lexer.BracketLeftToken(0, 0),
            lexer.EndToken(0, 1)
        ])
        self.assertEqual(lex('a'), [
            lexer.TextToken(0, 0, 'a'),
            lexer.EndToken(0, 1)
        ])


    def test_text_escaping(self):
        input_str = r'abcd\\ fooey \[ \]'
        expected_output_str = r'abcd\ fooey [ ]'
        self.assertEqual(lex(input_str), [
            lexer.TextToken(0, 0, expected_output_str),
            # End token comes at the end of the *input* string!
            lexer.EndToken(0, len(input_str))
        ])

    def test_stuff_that_fails_outside_brackets(self):
        with self.assertRaises(lexer.LexError):
            lex(']')
<commit_msg>More tests<commit_after>import unittest

from kissup import lexer


def lex(s):
    return list(lexer.lex_kissup(s))


class TestLexer(unittest.TestCase):
    def test_single_tokens_outside_brackets(self):
        self.assertEqual(lex('['), [
            lexer.BracketLeftToken(0, 0),
            lexer.EndToken(0, 1)
        ])
        self.assertEqual(lex('a'), [
            lexer.TextToken(0, 0, 'a'),
            lexer.EndToken(0, 1)
        ])


    def test_text_escaping(self):
        input_str = r'abcd\\ fooey \[ \]'
        expected_output_str = r'abcd\ fooey [ ]'
        self.assertEqual(lex(input_str), [
            lexer.TextToken(0, 0, expected_output_str),
            # End token comes at the end of the *input* string!
            lexer.EndToken(0, len(input_str))
        ])

    def test_bad_text_escapes(self):
        with self.assertRaises(lexer.LexError):
            lex(r'\z')
        with self.assertRaises(lexer.LexError):
            lex("\\")

    def test_stuff_that_fails_outside_brackets(self):
        with self.assertRaises(lexer.LexError):
            lex(']')

{'fraction_non_alphanumeric': 0.12549407114624506, 'fraction_numerical': 0.010869565217391304, 'mean_word_length': 2.2873376623376624, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2042712', 'n_tokens_mistral': 728, 'n_tokens_neox': 667, 'n_words': 117}