<commit_before>"""
A naive but strict HTML tokenizer. Based directly on
http://www.w3.org/TR/2011/WD-html5-20110525/tokenization.html

In the ATTRIBUTE_VALUE and BEFORE_ATTRIBUTE_VALUE states, python tokens are accepted.
"""

import sys
from collections import OrderedDict

class State(object):
    DATA = 1
    # unused states: charrefs, RCDATA, script, RAWTEXT, PLAINTEXT
    TAG_OPEN = 7
    END_TAG_OPEN = 8
    TAG_NAME = 9
    # unused states: RCDATA, RAWTEXT, script
    BEFORE_ATTRIBUTE_NAME = 34
    ATTRIBUTE_NAME = 35
    AFTER_ATTRIBUTE_NAME = 36
    BEFORE_ATTRIBUTE_VALUE = 37
    ATTRIBUTE_VALUE_DOUBLE_QUOTED = 38
    ATTRIBUTE_VALUE_SINGLE_QUOTED = 39
    ATTRIBUTE_VALUE_UNQUOTED = 40
    # unused state: CHARREF_IN_ATTRIBUTE_VALUE = 41
    AFTER_ATTRIBUTE_VALUE = 42
    SELF_CLOSING_START_TAG = 43
    # unused state: BOGUS_COMMENT_STATE = 44
    MARKUP_DECLARATION_OPEN = 45
    COMMENT_START = 46
    COMMENT_START_DASH = 47
    COMMENT = 48
    COMMENT_END_DASH = 49
    COMMENT_END = 50
    # unused state: COMMENT_END_BANG = 51
    DOCTYPE = 52
    DOCTYPE_CONTENTS = 53 # Gross oversimplification. Not to spec.
    # unused states: doctypes
    CDATA_SECTION = 68

    @classmethod
    def state_name(cls, state_val):
        for k, v in cls.__dict__.items():
            if v == state_val:
                return k
        assert False, "impossible state value %r!" % state_val

class Tag(object):
    def __init__(self):
        self.tag_name = None
        self.attrs = OrderedDict()
        self.endtag = False
        self.startendtag = False

class ParseError(Exception):
    pass

class BadCharError(Exception):
    def __init__(self, state, char):
        super(BadCharError, self).__init__("unexpected character %r in state %r" %
                                           (char, State.state_name(state)))

class Unimplemented(Exception):
    pass

class HTMLTokenizer(object):

    def __init__(self):
        self.state = State.DATA

        # attribute_value is a list, where each element is either a string or a list of python
        # tokens.

        self.data = ""
        self.tag = None
        self.tag_name = None
        self.attribute_name = None
        self.attribute_value = None
        self.markup_declaration_buffer = None

    def handle_data(self, data):
        assert False, "subclass should override"

    def handle_starttag(self, tag_name, attrs):
        assert False, "subclass should override"

    def handle_startendtag(self, tag_name, attrs):
        assert False, "subclass should override"

    def handle_endtag(self, tag_name):
        assert False, "subclass should override"

    def handle_comment(self, tag_name):
        assert False, "subclass should override"

    def handle_doctype(self, data):
        assert False, "subclass should override"

    def handle_cdata(self, tag_name):
        assert False, "subclass should override"

    def emit_data(self):
        self.handle_data(self.data)
        self.data = ""

    def emit_tag(self):
        if self.tag.startendtag and self.tag.endtag:
            raise ParseError("both startendtag and endtag!?")
        if self.tag.startendtag:
            self.handle_startendtag(self.tag.tag_name, self.tag.attrs)
        elif self.tag.endtag:
            self.handle_endtag(self.tag.tag_name)
        else:
            self.handle_starttag(self.tag.tag_name, self.tag.attrs)

    def emit_comment(self):
        self.handle_comment(self.data)
        self.data = ""
<commit_msg>Do not consider assertions for coverage metrics.<commit_after>"""
A naive but strict HTML tokenizer. Based directly on
http://www.w3.org/TR/2011/WD-html5-20110525/tokenization.html

In the ATTRIBUTE_VALUE and BEFORE_ATTRIBUTE_VALUE states, python tokens are accepted.
"""

import sys
from collections import OrderedDict

class State(object):
    DATA = 1
    # unused states: charrefs, RCDATA, script, RAWTEXT, PLAINTEXT
    TAG_OPEN = 7
    END_TAG_OPEN = 8
    TAG_NAME = 9
    # unused states: RCDATA, RAWTEXT, script
    BEFORE_ATTRIBUTE_NAME = 34
    ATTRIBUTE_NAME = 35
    AFTER_ATTRIBUTE_NAME = 36
    BEFORE_ATTRIBUTE_VALUE = 37
    ATTRIBUTE_VALUE_DOUBLE_QUOTED = 38
    ATTRIBUTE_VALUE_SINGLE_QUOTED = 39
    ATTRIBUTE_VALUE_UNQUOTED = 40
    # unused state: CHARREF_IN_ATTRIBUTE_VALUE = 41
    AFTER_ATTRIBUTE_VALUE = 42
    SELF_CLOSING_START_TAG = 43
    # unused state: BOGUS_COMMENT_STATE = 44
    MARKUP_DECLARATION_OPEN = 45
    COMMENT_START = 46
    COMMENT_START_DASH = 47
    COMMENT = 48
    COMMENT_END_DASH = 49
    COMMENT_END = 50
    # unused state: COMMENT_END_BANG = 51
    DOCTYPE = 52
    DOCTYPE_CONTENTS = 53 # Gross oversimplification. Not to spec.
    # unused states: doctypes
    CDATA_SECTION = 68

    @classmethod
    def state_name(cls, state_val):
        for k, v in cls.__dict__.items():
            if v == state_val:
                return k
        # TODO: This should be an exception rather than an assertion.
        assert False, "impossible state value %r!" % state_val # pragma: no cover

class Tag(object):
    def __init__(self):
        self.tag_name = None
        self.attrs = OrderedDict()
        self.endtag = False
        self.startendtag = False

class ParseError(Exception):
    pass

class BadCharError(Exception):
    def __init__(self, state, char):
        super(BadCharError, self).__init__("unexpected character %r in state %r" %
                                           (char, State.state_name(state)))

class Unimplemented(Exception):
    pass

class HTMLTokenizer(object):

    def __init__(self):
        self.state = State.DATA

        # attribute_value is a list, where each element is either a string or a list of python
        # tokens.

        self.data = ""
        self.tag = None
        self.tag_name = None
        self.attribute_name = None
        self.attribute_value = None
        self.markup_declaration_buffer = None

    # TODO: There are better ways to implement this. Abstract Base Class or
    # at least raising NotImplementedError instead of asserting in the below
    # methods.

    def handle_data(self, data): # pragma: no cover
        assert False, "subclass should override"

    def handle_starttag(self, tag_name, attrs): # pragma: no cover
        assert False, "subclass should override"

    def handle_startendtag(self, tag_name, attrs): # pragma: no cover
        assert False, "subclass should override"

    def handle_endtag(self, tag_name): # pragma: no cover
        assert False, "subclass should override"

    def handle_comment(self, tag_name): # pragma: no cover
        assert False, "subclass should override"

    def handle_doctype(self, data): # pragma: no cover
        assert False, "subclass should override"

    def handle_cdata(self, tag_name): # pragma: no cover
        assert False, "subclass should override"

    def emit_data(self):
        self.handle_data(self.data)
        self.data = ""

    def emit_tag(self):
        if self.tag.startendtag and self.tag.endtag:
            raise ParseError("both startendtag and endtag!?")
        if self.tag.startendtag:
            self.handle_startendtag(self.tag.tag_name, self.tag.attrs)
        elif self.tag.endtag:
            self.handle_endtag(self.tag.tag_name)
        else:
            self.handle_starttag(self.tag.tag_name, self.tag.attrs)

    def emit_comment(self):
        self.handle_comment(self.data)
        self.data = ""

{'fraction_non_alphanumeric': 0.0793010752688172, 'fraction_numerical': 0.016129032258064516, 'mean_word_length': 2.674567901234568, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2618125', 'n_tokens_mistral': 2442, 'n_tokens_neox': 2279, 'n_words': 623}