<commit_before>try:
    from sets import ImmutableSet as frozenset
except:
    pass

from constants import contentModelFlags, spaceCharacters
from constants import entitiesWindows1252, entities, voidElements
from constants import asciiLowercase, asciiUppercase, asciiLetters
from constants import digits, hexDigits

# Data representing the end of the input stream. Needs to be a string for
# comparing: "x in string" etc...
EOF = u"\x00"

# Token objects used to hold token data when tokens are in the
# process of being constructed
class Token(object):
    """Abstract base class from which all tokens derive
    """
    def __init__(self):
        raise NotImplementedError

class DoctypeToken(Token):
    """Token representing a DOCTYPE
    Attributes - name:  The name of the doctype
                 error: The Error status of the doctype)
    """
    def __init__(self, name=None):
        self.name = name
        self.error = True

class TagToken(Token):
    """Token representing a tag.<commit_msg>Revert EOF marker to None<commit_after>try:
    from sets import ImmutableSet as frozenset
except:
    pass

from constants import contentModelFlags, spaceCharacters
from constants import entitiesWindows1252, entities, voidElements
from constants import asciiLowercase, asciiUppercase, asciiLetters
from constants import digits, hexDigits

# Data representing the end of the input stream. Needs to be a string for
# comparing: "x in string" etc...
EOF = None

# Token objects used to hold token data when tokens are in the
# process of being constructed
class Token(object):
    """Abstract base class from which all tokens derive
    """
    def __init__(self):
        raise NotImplementedError

class DoctypeToken(Token):
    """Token representing a DOCTYPE
    Attributes - name:  The name of the doctype
                 error: The Error status of the doctype)
    """
    def __init__(self, name=None):
        self.name = name
        self.error = True

class TagToken(Token):
    """Token representing a tag.
{'fraction_non_alphanumeric': 0.06475531389026198, 'fraction_numerical': 0.004943153732081068, 'mean_word_length': 3.5585585585585586, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '290805', 'n_tokens_mistral': 548, 'n_tokens_neox': 526, 'n_words': 233}