<commit_before># Differences from the current specification (23 December 2006) are as follows:
# * Phases and insertion modes are one concept in parser.py.
# * EOF handling is slightly different to make sure <html>, <head> and <body>
#   always exist.
# * We also deal with content when there's no DOCTYPE.
# It is expected that the specification will catch up with us in due course ;-)
#
# It should be trivial to add the following cases. However, we should probably
# also look into comment handling and such then...
# * A <p> element end tag creates an empty <p> element when there's no <p>
#   element in scope.
# * A <br> element end tag creates an empty <br> element.

try:
    frozenset
except NameError:
    # Import from the sets module for python 2.3
    from sets import Set as set
    from sets import ImmutableSet as frozenset
import gettext
_ = gettext.gettext

import tokenizer

import treebuilders
from treebuilders._base import Marker
from treebuilders import simpletree

import utils
from constants import contentModelFlags, spaceCharacters, asciiUpper2Lower
from constants import scopingElements, formattingElements, specialElements
from constants import headingElements, tableInsertModeElements, voidElements

class HTMLParser(object):
    """HTML parser. Generates a tree structure from a stream of (possibly
        malformed) HTML"""

    def __init__(self, strict = False, tree=simpletree.TreeBuilder):
        """
        strict - raise an exception when a parse error is encountered 
        
        tree - a treebuilder class controlling the type of tree that will be 
        returned. This class is almost always a subclass of 
        html5lib.treebuilders._base.TreeBuilder
        """

        # Raise an exception on the first error encountered
        self.strict = strict

        self.tree = tree()
        self.errors = []

        self.phases = {
            "initial": InitialPhase(self, self.tree),
            "rootElement": RootElementPhase(self, self.tree),
            "beforeHead": BeforeHeadPhase(self, self.tree),
            "inHead": InHeadPhase(self, self.tree),
            "afterHead": AfterHeadPhase(self, self.tree),
            "inBody": InBodyPhase(self, self.tree),
            "inTable": InTablePhase(self, self.tree),
            "inCaption": InCaptionPhase(self, self.tree),
            "inColumnGroup": InColumnGroupPhase(self, self.tree),
            "inTableBody": InTableBodyPhase(self, self.tree),
            "inRow": InRowPhase(self, self.tree),
            "inCell": InCellPhase(self, self.tree),
            "inSelect": InSelectPhase(self, self.tree),
            "afterBody": AfterBodyPhase(self, self.tree),
            "inFrameset": InFramesetPhase(self, self.tree),
            "afterFrameset": AfterFramesetPhase(self, self.tree),
            "trailingEnd": TrailingEndPhase(self, self.tree)
        }

    def parse(self, stream, encoding=None, innerHTML=False):
        """Parse a HTML document into a well-formed tree
        
        stream - a filelike object or string containing the HTML to be parsed
        
        innerHTML - Are we parsing in innerHTML mode (note innerHTML=True 
        is not yet supported)

        The optional encoding parameter must be a string that indicates
        the encoding.  If specified, that encoding will be used,
        regardless of any BOM or later declaration (such as in a meta
        element)
        """

        self.tree.reset()
        self.errors = []

        self.phase = self.phases["initial"]
        # We only seem to have InBodyPhase testcases where the following is
        # relevant ... need others too
        self.lastPhase = None

        # We don't actually support innerHTML yet but this should allow
        # assertations
        self.innerHTML = innerHTML

        self.tokenizer = tokenizer.HTMLTokenizer(stream, encoding)

        # XXX This is temporary for the moment so there isn't any other
        # changes needed for the parser to work with the iterable tokenizer
        for token in self.tokenizer:
            token = self.normalizeToken(token)
            type = token["type"]
            method = getattr(self.phase, "process%s" % type, None)
            if type in ("Characters", "SpaceCharacters", "Comment"):
                method(token["data"])
            elif type in ("StartTag", "Doctype"):
                method(token["name"], token["data"])
            elif type == "EndTag":
                method(token["name"])
            else:
                self.parseError(token["data"])

        # When the loop finishes it's EOF
        self.phase.processEOF()

        return self.tree.getDocument()

    def parseError(self, data="XXX ERROR MESSAGE NEEDED"):
        # The idea is to make data mandatory.
        self.errors.append((self.tokenizer.stream.position(), data))
        if self.strict:
            raise ParseError

    def atheistParseError(self):
        """This error is not an error"""
        pass

    def normalizeToken(self, token):
        """ HTML5 specific normalizations to the token stream """
       
        if token["type"] == "EmptyTag":
            # When a solidus (/) is encountered within a tag name what happens
            # depends on whether the current tag name matches that of a void
            # element.  If it matches a void element atheists did the wrong
            # thing and if it doesn't it's wrong for everyone.

            if token["name"] in voidElements:
                self.atheistParseError()
            else:
                self.parseError(_("Solidus (/) incorrectly placed in tag."))

            token["type"] = "StartTag"

        if token["type"] == "StartTag":
            token["name"] = token["name"].translate(asciiUpper2Lower)

            # We need to remove the duplicate attributes and convert attributes
            # to a dict so that [["x", "y"], ["x", "z"]] becomes {"x": "y"}

            # AT When Python 2.4 is widespread we should use
            # dict(reversed(token.data))
            if token["data"]:
                token["data"] = dict([(attr.translate(asciiUpper2Lower), value)
                    for attr,value in token["data"][::-1]])
            else:
                token["data"] = {}

        elif token["type"] == "EndTag":
            if token["data"]:
               self.parseError(_("End tag contains unexpected attributes."))
               self.token["data"] = {}
            token["name"] = token["name"].lower()

        return token

    #XXX - almost everthing after this point should be moved into a
    #seperate treebuilder object


    def resetInsertionMode(self):
        # The name of this method is mostly historical. (It's also used in the
        # specification.)
        last = False
        newModes = {
            "select":"inSelect",
            "td":"inCell",
            "th":"inCell",
            "tr":"inRow",
            "tbody":"inTableBody",
            "thead":"inTableBody",
            "tfoot":"inTableBody",
            "caption":"inCaption",
            "colgroup":"inColumnGroup",
            "table":"inTable",
            "head":"inBody",
            "body":"inBody",
            "frameset":"inFrameset"
        }
        for node in self.tree.openElements[::-1]:<commit_msg>fix small bug in html5parser.py<commit_after># Differences from the current specification (23 December 2006) are as follows:
# * Phases and insertion modes are one concept in parser.py.
# * EOF handling is slightly different to make sure <html>, <head> and <body>
#   always exist.
# * We also deal with content when there's no DOCTYPE.
# It is expected that the specification will catch up with us in due course ;-)
#
# It should be trivial to add the following cases. However, we should probably
# also look into comment handling and such then...
# * A <p> element end tag creates an empty <p> element when there's no <p>
#   element in scope.
# * A <br> element end tag creates an empty <br> element.

try:
    frozenset
except NameError:
    # Import from the sets module for python 2.3
    from sets import Set as set
    from sets import ImmutableSet as frozenset
import gettext
_ = gettext.gettext

import tokenizer

import treebuilders
from treebuilders._base import Marker
from treebuilders import simpletree

import utils
from constants import contentModelFlags, spaceCharacters, asciiUpper2Lower
from constants import scopingElements, formattingElements, specialElements
from constants import headingElements, tableInsertModeElements, voidElements

class HTMLParser(object):
    """HTML parser. Generates a tree structure from a stream of (possibly
        malformed) HTML"""

    def __init__(self, strict = False, tree=simpletree.TreeBuilder):
        """
        strict - raise an exception when a parse error is encountered 
        
        tree - a treebuilder class controlling the type of tree that will be 
        returned. This class is almost always a subclass of 
        html5lib.treebuilders._base.TreeBuilder
        """

        # Raise an exception on the first error encountered
        self.strict = strict

        self.tree = tree()
        self.errors = []

        self.phases = {
            "initial": InitialPhase(self, self.tree),
            "rootElement": RootElementPhase(self, self.tree),
            "beforeHead": BeforeHeadPhase(self, self.tree),
            "inHead": InHeadPhase(self, self.tree),
            "afterHead": AfterHeadPhase(self, self.tree),
            "inBody": InBodyPhase(self, self.tree),
            "inTable": InTablePhase(self, self.tree),
            "inCaption": InCaptionPhase(self, self.tree),
            "inColumnGroup": InColumnGroupPhase(self, self.tree),
            "inTableBody": InTableBodyPhase(self, self.tree),
            "inRow": InRowPhase(self, self.tree),
            "inCell": InCellPhase(self, self.tree),
            "inSelect": InSelectPhase(self, self.tree),
            "afterBody": AfterBodyPhase(self, self.tree),
            "inFrameset": InFramesetPhase(self, self.tree),
            "afterFrameset": AfterFramesetPhase(self, self.tree),
            "trailingEnd": TrailingEndPhase(self, self.tree)
        }

    def parse(self, stream, encoding=None, innerHTML=False):
        """Parse a HTML document into a well-formed tree
        
        stream - a filelike object or string containing the HTML to be parsed
        
        innerHTML - Are we parsing in innerHTML mode (note innerHTML=True 
        is not yet supported)

        The optional encoding parameter must be a string that indicates
        the encoding.  If specified, that encoding will be used,
        regardless of any BOM or later declaration (such as in a meta
        element)
        """

        self.tree.reset()
        self.errors = []

        self.phase = self.phases["initial"]
        # We only seem to have InBodyPhase testcases where the following is
        # relevant ... need others too
        self.lastPhase = None

        # We don't actually support innerHTML yet but this should allow
        # assertations
        self.innerHTML = innerHTML

        self.tokenizer = tokenizer.HTMLTokenizer(stream, encoding)

        # XXX This is temporary for the moment so there isn't any other
        # changes needed for the parser to work with the iterable tokenizer
        for token in self.tokenizer:
            token = self.normalizeToken(token)
            type = token["type"]
            method = getattr(self.phase, "process%s" % type, None)
            if type in ("Characters", "SpaceCharacters", "Comment"):
                method(token["data"])
            elif type in ("StartTag", "Doctype"):
                method(token["name"], token["data"])
            elif type == "EndTag":
                method(token["name"])
            else:
                self.parseError(token["data"])

        # When the loop finishes it's EOF
        self.phase.processEOF()

        return self.tree.getDocument()

    def parseError(self, data="XXX ERROR MESSAGE NEEDED"):
        # The idea is to make data mandatory.
        self.errors.append((self.tokenizer.stream.position(), data))
        if self.strict:
            raise ParseError

    def atheistParseError(self):
        """This error is not an error"""
        pass

    def normalizeToken(self, token):
        """ HTML5 specific normalizations to the token stream """
       
        if token["type"] == "EmptyTag":
            # When a solidus (/) is encountered within a tag name what happens
            # depends on whether the current tag name matches that of a void
            # element.  If it matches a void element atheists did the wrong
            # thing and if it doesn't it's wrong for everyone.

            if token["name"] in voidElements:
                self.atheistParseError()
            else:
                self.parseError(_("Solidus (/) incorrectly placed in tag."))

            token["type"] = "StartTag"

        if token["type"] == "StartTag":
            token["name"] = token["name"].translate(asciiUpper2Lower)

            # We need to remove the duplicate attributes and convert attributes
            # to a dict so that [["x", "y"], ["x", "z"]] becomes {"x": "y"}

            # AT When Python 2.4 is widespread we should use
            # dict(reversed(token.data))
            if token["data"]:
                token["data"] = dict([(attr.translate(asciiUpper2Lower), value)
                    for attr,value in token["data"][::-1]])
            else:
                token["data"] = {}

        elif token["type"] == "EndTag":
            if token["data"]:
               self.parseError(_("End tag contains unexpected attributes."))
               token["data"] = {}
            token["name"] = token["name"].lower()

        return token

    #XXX - almost everthing after this point should be moved into a
    #seperate treebuilder object


    def resetInsertionMode(self):
        # The name of this method is mostly historical. (It's also used in the
        # specification.)
        last = False
        newModes = {
            "select":"inSelect",
            "td":"inCell",
            "th":"inCell",
            "tr":"inRow",
            "tbody":"inTableBody",
            "thead":"inTableBody",
            "tfoot":"inTableBody",
            "caption":"inCaption",
            "colgroup":"inColumnGroup",
            "table":"inTable",
            "head":"inBody",
            "body":"inBody",
            "frameset":"inFrameset"
        }
        for node in self.tree.openElements[::-1]:
{'fraction_non_alphanumeric': 0.09930731774226734, 'fraction_numerical': 0.002400384061449832, 'mean_word_length': 2.2643832549809715, 'pattern_counts': {'":': 70, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1157616', 'n_tokens_mistral': 3863, 'n_tokens_neox': 3700, 'n_words': 1331}