# Path: purplequery/tokenizer.py
# def remove_comments(query):
#     # type: (str) -> str
#     return _COMPILED_COMMENT_PATTERN.sub('', query)
# 
# def tokenize(query):
#     # type: (str) -> List[str]
#     return _COMBINED_PATTERN.findall(remove_comments(query))


import unittest
from purplequery.tokenizer import remove_comments, tokenize
class TokenizerTest(unittest.TestCase):

    def test_remove_comments(self):
        # type: () -> None
        """Tests removal of comments"""
        self.assertEqual(
            remove_comments(
                'foo # end-of-line-with-pound\n'
                '# successive lines\n'
                '# of nothing but comments\n'
                'more and -- end-of-line-with --\n'
                '-- successive lines\n'
                '-- of nothing but comments\n'
                'with /* comments in the middle */ removed\n'
                'even /* comments *//* that *//* abut */ removed\n'
                'but /* does not end * / yet * still not */ now\n'),
            ('foo \n'
             '\n'
             '\n'
             'more and \n'
             '\n'
             '\n'
             'with  removed\n'
             'even  removed\n'
             'but  now\n'))

    def test_general_tokenization(self):
        # type: () -> None
        """Tests tokenization of numbers, strings, variables, comments, lists."""
        self.assertEqual(

# Path: purplequery/tokenizer.py
# def remove_comments(query):
#     # type: (str) -> str
#     return _COMPILED_COMMENT_PATTERN.sub('', query)
# 
# def tokenize(query):
#     # type: (str) -> List[str]
#     return _COMBINED_PATTERN.findall(remove_comments(query))


# Path: purplequery/tokenizer_test.py
import unittest
from purplequery.tokenizer import remove_comments, tokenize

class TokenizerTest(unittest.TestCase):

    def test_remove_comments(self):
        # type: () -> None
        """Tests removal of comments"""
        self.assertEqual(
            remove_comments(
                'foo # end-of-line-with-pound\n'
                '# successive lines\n'
                '# of nothing but comments\n'
                'more and -- end-of-line-with --\n'
                '-- successive lines\n'
                '-- of nothing but comments\n'
                'with /* comments in the middle */ removed\n'
                'even /* comments *//* that *//* abut */ removed\n'
                'but /* does not end * / yet * still not */ now\n'),
            ('foo \n'
             '\n'
             '\n'
             'more and \n'
             '\n'
             '\n'
             'with  removed\n'
             'even  removed\n'
             'but  now\n'))

    def test_general_tokenization(self):
        # type: () -> None
        """Tests tokenization of numbers, strings, variables, comments, lists."""
        self.assertEqual(

            tokenize(