<commit_before># Differences from the current specification (23 December 2006) are as follows:
# * Phases and insertion modes are one concept in parser.py.
# * EOF handling is slightly different to make sure <html>, <head> and <body>
#   always exist.
# * We also deal with content when there's no DOCTYPE.
# It is expected that the specification will catch up with us in due course ;-)
#
# It should be trivial to add the following cases. However, we should probably
# also look into comment handling and such then...
# * A <p> element end tag creates an empty <p> element when there's no <p>
#   element in scope.
# * A <br> element end tag creates an empty <br> element.

try:
    frozenset
except NameError:
    # Import from the sets module for python 2.3
    from sets import Set as set
    from sets import ImmutableSet as frozenset
import gettext
_ = gettext.gettext

import tokenizer

import treebuilders
from treebuilders._base import Marker
from treebuilders import simpletree

import utils
from constants import contentModelFlags, spaceCharacters
from constants import scopingElements, formattingElements, specialElements
from constants import headingElements, tableInsertModeElements

class HTMLParser(object):
    """Main parser class"""

    def __init__(self, strict = False, tree=simpletree.TreeBuilder):
        """HTML parser. Generates a tree structure from a stream of (possibly
        malformed) HTML.  
        strict - raise an exception when a parse error is encountered 
        tree - a treebuilder class controlling the type of tree that will be 
        returned (default - html5lib.simpletree.TreeBuilder)"""

        # Raise an exception on the first error encountered
        self.strict = strict
        self.errors = []

        self._treeCls = tree

        self.phases = {
            "initial": InitialPhase(self, self.tree),
            "rootElement": RootElementPhase(self, self.tree),
            "beforeHead": BeforeHeadPhase(self, self.tree),
            "inHead": InHeadPhase(self, self.tree),
            "afterHead": AfterHeadPhase(self, self.tree),
            "inBody": InBodyPhase(self, self.tree),
            "inTable": InTablePhase(self, self.tree),
            "inCaption": InCaptionPhase(self, self.tree),
            "inColumnGroup": InColumnGroupPhase(self, self.tree),
            "inTableBody": InTableBodyPhase(self, self.tree),
            "inRow": InRowPhase(self, self.tree),
            "inCell": InCellPhase(self, self.tree),
            "inSelect": InSelectPhase(self, self.tree),
            "afterBody": AfterBodyPhase(self, self.tree),
            "inFrameset": InFramesetPhase(self, self.tree),
            "afterFrameset": AfterFramesetPhase(self, self.tree),
            "trailingEnd": TrailingEndPhase(self, self.tree)
        }
        self.phase = self.phases["initial"]
        # We only seem to have InBodyPhase testcases where the following is
        # relevant ... need others too
        self.lastPhase = None

    def parse(self, stream, innerHTML=False):
        """Stream should be a stream of unicode bytes. Character encoding
        issues have not yet been dealt with."""

        # XXX - need to ensure the tree is reset here
        self.tree = self._treeCls()

        # We don't actually support innerHTML yet but this should allow
        # assertations
        self.innerHTML = innerHTML

        self.tokenizer = tokenizer.HTMLTokenizer(stream)

        # XXX This is temporary for the moment so there isn't any other
        # changes needed for the parser to work with the iterable tokenizer
        for token in self.tokenizer:<commit_msg>Revert disaterous change that prevented the parser from running<commit_after># Differences from the current specification (23 December 2006) are as follows:
# * Phases and insertion modes are one concept in parser.py.
# * EOF handling is slightly different to make sure <html>, <head> and <body>
#   always exist.
# * We also deal with content when there's no DOCTYPE.
# It is expected that the specification will catch up with us in due course ;-)
#
# It should be trivial to add the following cases. However, we should probably
# also look into comment handling and such then...
# * A <p> element end tag creates an empty <p> element when there's no <p>
#   element in scope.
# * A <br> element end tag creates an empty <br> element.

try:
    frozenset
except NameError:
    # Import from the sets module for python 2.3
    from sets import Set as set
    from sets import ImmutableSet as frozenset
import gettext
_ = gettext.gettext

import tokenizer

import treebuilders
from treebuilders._base import Marker
from treebuilders import simpletree

import utils
from constants import contentModelFlags, spaceCharacters
from constants import scopingElements, formattingElements, specialElements
from constants import headingElements, tableInsertModeElements

class HTMLParser(object):
    """Main parser class"""

    def __init__(self, strict = False, tree=simpletree.TreeBuilder):
        """HTML parser. Generates a tree structure from a stream of (possibly
        malformed) HTML.  
        strict - raise an exception when a parse error is encountered 
        tree - a treebuilder class controlling the type of tree that will be 
        returned (default - html5lib.simpletree.TreeBuilder)"""

        # Raise an exception on the first error encountered
        self.strict = strict
        self.errors = []

        self.tree = tree()

        self.phases = {
            "initial": InitialPhase(self, self.tree),
            "rootElement": RootElementPhase(self, self.tree),
            "beforeHead": BeforeHeadPhase(self, self.tree),
            "inHead": InHeadPhase(self, self.tree),
            "afterHead": AfterHeadPhase(self, self.tree),
            "inBody": InBodyPhase(self, self.tree),
            "inTable": InTablePhase(self, self.tree),
            "inCaption": InCaptionPhase(self, self.tree),
            "inColumnGroup": InColumnGroupPhase(self, self.tree),
            "inTableBody": InTableBodyPhase(self, self.tree),
            "inRow": InRowPhase(self, self.tree),
            "inCell": InCellPhase(self, self.tree),
            "inSelect": InSelectPhase(self, self.tree),
            "afterBody": AfterBodyPhase(self, self.tree),
            "inFrameset": InFramesetPhase(self, self.tree),
            "afterFrameset": AfterFramesetPhase(self, self.tree),
            "trailingEnd": TrailingEndPhase(self, self.tree)
        }
        self.phase = self.phases["initial"]
        # We only seem to have InBodyPhase testcases where the following is
        # relevant ... need others too
        self.lastPhase = None

    def parse(self, stream, innerHTML=False):
        """Stream should be a stream of unicode bytes. Character encoding
        issues have not yet been dealt with."""

        # XXX - need to ensure the tree is reset here
        

        # We don't actually support innerHTML yet but this should allow
        # assertations
        self.innerHTML = innerHTML

        self.tokenizer = tokenizer.HTMLTokenizer(stream)

        # XXX This is temporary for the moment so there isn't any other
        # changes needed for the parser to work with the iterable tokenizer
        for token in self.tokenizer:
{'fraction_non_alphanumeric': 0.08158508158508158, 'fraction_numerical': 0.0024681201151789387, 'mean_word_length': 2.8942872397223707, 'pattern_counts': {'":': 34, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1945019', 'n_tokens_mistral': 1940, 'n_tokens_neox': 1879, 'n_words': 743}