# Path: cpp/tokenize.py
# VALID_IDENTIFIER_FIRST_CHARS = frozenset(_valid_identifier_first_char)
# VALID_IDENTIFIER_CHARS = frozenset(_valid_identifier_char)
# HEX_DIGITS = frozenset('0123456789abcdefABCDEF')
# INT_OR_FLOAT_DIGITS = frozenset('01234567890eE-+')
# _STR_PREFIXES = frozenset(('R', 'u8', 'u8R', 'u', 'uR', 'U', 'UR', 'L', 'LR'))
# UNKNOWN = 'UNKNOWN'
# SYNTAX = 'SYNTAX'
# CONSTANT = 'CONSTANT'
# NAME = 'NAME'
# PREPROCESSOR = 'PREPROCESSOR'
# class TokenError(Exception):
# class Token(object):
#     def __init__(self, token_type, name, start, end):
#     def __str__(self):
# def _get_string(source, i):
# def _get_char(source, start, i):
# def get_tokens(source):
# def _find(string, sub_string, start_index):


import unittest
from cpp import tokenize
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tokenize test."""

from __future__ import absolute_import




__author__ = 'nnorwitz@google.com (Neal Norwitz)'


# For convenience, add factories and __eq__ to test the module.


def Syntax(name, start, end):
    return tokenize.Token(tokenize.SYNTAX, name, start, end)


def Constant(name, start, end):
    return tokenize.Token(tokenize.CONSTANT, name, start, end)


def Name(name, start, end):
    return tokenize.Token(tokenize.NAME, name, start, end)


def Preprocessor(name, start, end):

# Path: cpp/tokenize.py
# VALID_IDENTIFIER_FIRST_CHARS = frozenset(_valid_identifier_first_char)
# VALID_IDENTIFIER_CHARS = frozenset(_valid_identifier_char)
# HEX_DIGITS = frozenset('0123456789abcdefABCDEF')
# INT_OR_FLOAT_DIGITS = frozenset('01234567890eE-+')
# _STR_PREFIXES = frozenset(('R', 'u8', 'u8R', 'u', 'uR', 'U', 'UR', 'L', 'LR'))
# UNKNOWN = 'UNKNOWN'
# SYNTAX = 'SYNTAX'
# CONSTANT = 'CONSTANT'
# NAME = 'NAME'
# PREPROCESSOR = 'PREPROCESSOR'
# class TokenError(Exception):
# class Token(object):
#     def __init__(self, token_type, name, start, end):
#     def __str__(self):
# def _get_string(source, i):
# def _get_char(source, start, i):
# def get_tokens(source):
# def _find(string, sub_string, start_index):


# Path: test_tokenize.py
import unittest
from cpp import tokenize

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Tokenize test."""

from __future__ import absolute_import




__author__ = 'nnorwitz@google.com (Neal Norwitz)'


# For convenience, add factories and __eq__ to test the module.


def Syntax(name, start, end):
    return tokenize.Token(tokenize.SYNTAX, name, start, end)


def Constant(name, start, end):
    return tokenize.Token(tokenize.CONSTANT, name, start, end)


def Name(name, start, end):
    return tokenize.Token(tokenize.NAME, name, start, end)


def Preprocessor(name, start, end):

    return tokenize.Token(tokenize.PREPROCESSOR, name, start, end)