# Path: textplusstuff/exceptions.py
# class MalformedToken(Exception):
#     pass
# 
# Path: textplusstuff/parser/tokens.py
# class TextPlusStuffToken(object):
# 
#     def __init__(self, token_type, contents):
#         # token_type must be MARKDOWN_FLAVORED_TEXT_TOKEN
#         # or RICHTEXT_TOKEN.
#         self.token_type, self.contents = token_type, contents
#         self.lineno = None
# 
#     def __unicode__(self):
#         return '<{token_type} token: "{content}...">'.format(
#             token_type=self.token_type,
#             content=self.contents[:10].replace('\n', '')
#         )


import re
from ..exceptions import MalformedToken
from .tokens import TextPlusStuffToken
        # is at the beginning of `self.raw_val` the first value
        # returned will be an empty string.
        in_tag = False
        result = []
        for chunk in textplusstuff_re.split(self.raw_val):
            # Ensures empty, non-new-line values (i.e. '') won't
            # create unnecessary tokens
            if chunk:
                result.append(self.create_token(chunk, in_tag))
            # Since `textplusstuff_re.split(self.raw_val)` will return
            # alternating token types `in_tag` must switch between
            # `True` and `False` during each iteration.
            in_tag = not in_tag
        return result

    def create_token(self, token_string, in_tag):
        """
        Convert the given token string into a new Token object and return it.
        If `in_tag` is `True`, we are processing a 'chunk' that matched the
        pattern in `textplusstuff_re`, otherwise it should be treated as a
        markdown-flavored string.
        """
        if in_tag:
            try:
                token_type, token_string = token_string.split('__')
            except ValueError:
                raise MalformedToken(
                    "{} is a malformed token string!".format(token_string)
                )
            else:

# Path: textplusstuff/exceptions.py
# class MalformedToken(Exception):
#     pass
# 
# Path: textplusstuff/parser/tokens.py
# class TextPlusStuffToken(object):
# 
#     def __init__(self, token_type, contents):
#         # token_type must be MARKDOWN_FLAVORED_TEXT_TOKEN
#         # or RICHTEXT_TOKEN.
#         self.token_type, self.contents = token_type, contents
#         self.lineno = None
# 
#     def __unicode__(self):
#         return '<{token_type} token: "{content}...">'.format(
#             token_type=self.token_type,
#             content=self.contents[:10].replace('\n', '')
#         )


# Path: textplusstuff/parser/lexer.py
import re
from ..exceptions import MalformedToken
from .tokens import TextPlusStuffToken

        # is at the beginning of `self.raw_val` the first value
        # returned will be an empty string.
        in_tag = False
        result = []
        for chunk in textplusstuff_re.split(self.raw_val):
            # Ensures empty, non-new-line values (i.e. '') won't
            # create unnecessary tokens
            if chunk:
                result.append(self.create_token(chunk, in_tag))
            # Since `textplusstuff_re.split(self.raw_val)` will return
            # alternating token types `in_tag` must switch between
            # `True` and `False` during each iteration.
            in_tag = not in_tag
        return result

    def create_token(self, token_string, in_tag):
        """
        Convert the given token string into a new Token object and return it.
        If `in_tag` is `True`, we are processing a 'chunk' that matched the
        pattern in `textplusstuff_re`, otherwise it should be treated as a
        markdown-flavored string.
        """
        if in_tag:
            try:
                token_type, token_string = token_string.split('__')
            except ValueError:
                raise MalformedToken(
                    "{} is a malformed token string!".format(token_string)
                )
            else:

                token = TextPlusStuffToken(