<commit_before>import tokenizer

from utils import utils
from constants import contentModelFlags, spaceCharacters
from constants import scopingElements, formattingElements, specialElements
from constants import headingElements, tableInsertModeElements
import inspect

"""The scope markers are inserted when entering buttons, object
elements, marquees, table cells, and table captions, and are used to
prevent formatting from "leaking" into tables, buttons, object
elements, and marquees."""
Marker = None

#Really crappy basic implementation of a DOM-core like thing
class Node(object):
    def __init__(self, name):
        self.name = name
        self.parent = None
        self.value = None
        self.childNodes = []
        self.attributes = {}
        self._flags = []

    def __str__(self):
        return self.name

    def __repr__(self):
        return "<%s %s>"%(self.__class__, self.name)

    def printTree(self, indent=0):
        tree = '\n|%s%s' % (' '*indent, str(self))
        for child in self.childNodes:
            tree += child.printTree(indent+2)
        return tree

    def appendChild(self, node, index=None):
        if (isinstance(node, TextNode) and self.childNodes and
          isinstance(self.childNodes[-1], TextNode)):
            self.childNodes[-1].value += node.value
        else:
            self.childNodes.append(node)
        node.parent = self

    def insertBefore(self, node, refNode):
        index = self.childNodes.index(refNode)
        if (isinstance(node, TextNode) and
          isinstance(self.childNodes[index - 1], TextNode)):
            self.childNodes[index - 1].value += node.value
        else:
            self.childNodes.insert(index, node)
        node.parent = self

    def removeChild(self, node):
        try:
            self.childNodes.remove(node)
        except:
            # XXX
            raise
        node.parent = None

    def cloneNode(self):
        newNode = type(self)(self.name)
        for attr, value in self.attributes.iteritems():
            newNode.attributes[attr] = value
        newNode.value = self.value
        return newNode

class Document(Node):
    def __init__(self):
        Node.__init__(self, None)

    def __str__(self):
        return '#document'

    def printTree(self):
        tree = str(self)
        for child in self.childNodes:
            tree += child.printTree(2)
        return tree

class DocumentType(Node):
    def __init__(self, name):
        Node.__init__(self, name)

    def __str__(self):
        return '<!DOCTYPE %s>' % self.name

class TextNode(Node):
    def __init__(self, value):
        Node.__init__(self, None)
        self.value = value

    def __str__(self):
        return '"%s"' % self.value

class Element(Node):
    def __init__(self, name):
        Node.__init__(self, name)

    def __str__(self):
        return '<%s>' % self.name

    def printTree(self, indent):
        tree = '\n|%s%s' % (' '*indent, str(self))
        indent += 2
        if self.attributes:
            for name, value in self.attributes.iteritems():
                tree += '\n|%s%s="%s"' % (' '*indent, name, value)
        for child in self.childNodes:
            tree += child.printTree(indent)
        return tree

class CommentNode(Node):
    def __init__(self, data):
        Node.__init__(self, None)
        self.data = data

    def __str__(self):
        return '<!-- %s -->' % self.data

class HTMLParser(object):
    """Main parser class"""

    def __init__(self, strict = False):
        # Raise an exception on the first error encountered
        self.strict = strict

        self.openElements = []
        self.activeFormattingElements = []
        self.headPointer = None
        self.formPointer = None

        self.phases = {"initial":InitialPhase,
                       "rootElement":RootElementPhase,
                       "main":MainPhase,
                       "trailingEnd":TrailingEndPhase}

    def parse(self, stream, innerHTML=False):
        """Stream should be a stream of unicode bytes. Character encoding
        issues have not yet been dealt with."""

        self.document = Document()

        #We don't actually support inner HTML yet but this should allow
        #assertations
        self.innerHTML = innerHTML

        #The parsing phase we are currently in
        self.phase = InitialPhase(self)

        #Flag indicationg special insertion mode from elements misnested inside
        #a table
        self.insertFromTable = False

        self.tokenizer = tokenizer.HTMLTokenizer(self)
        self.tokenizer.tokenize(stream)

        return self.document

    def processDoctype(self, name, error):
        self.phase.processDoctype(name, error)

    def processStartTag(self, name, attributes):
        self.phase.processStartTag(name, attributes)

    def processEndTag(self, name):
        self.phase.processEndTag(name)

    def processComment(self, data):
        self.phase.processComment(data)

    def processCharacter(self, data):
        self.phase.processCharacter(data)

    def processEOF(self):
        self.phase.processEOF()

    def parseError(self):
        if self.strict:
            raise ParseError

    def atheistParseError(self):
        """This error is not an error"""
        pass

    def switchPhase(self, name):
        """Switch between different phases of the parsing
        """
        # Need to hang on to state between trailing end phase and main phase
        if (name == "trailingEnd" and
            isinstance(self.phase, self.phases["main"])):
            self.mainPhaseState = self.phase
            self.phase = self.phases["trailingEnd"](self)
        elif (name == "main" and
              isinstance(self.phase, self.phases["trailingEnd"])):
            self.phase = self.mainPhaseState
        else:
            self.phase = self.phases[name](self)

    #XXX - almost everthing after this point should be moved into a
    #seperate treebuilder object

    def switchInsertionMode(self, name):
        """Switch between different insertion modes in the main phase"""
        # XXX AT Arguably this should be on the main phase object itself
        self.phase.insertionMode = self.phase.insertionModes[name](self)

    def elementInScope(self, target, tableVariant=False):
        # AT Use reverse instead of [::-1] when we can rely on Python 2.4
        # AT How about while True and simply set node to [-1] and set it to
        # [-2] at the end...
        for node in self.openElements[::-1]:
            if node.name == target:
                return True
            elif node.name == "table":
                return False
            elif not tableVariant and node.name in scopingElements:
                return False
            elif node.name == "html":
                return False
        assert False # We should never reach this point

    def reconstructActiveFormattingElements(self):
        # This covers the ", if any" case mentioned often in the drafts, before
        # you have to reconstruct the active formatting elements.
        if not self.activeFormattingElements:
            return

        # We start with the last element. So i is -1.
        i = -1
        entry = self.activeFormattingElements[i]

        # Step 1
        if entry == Marker or entry in self.openElements:
            return

        # Step 2 is covered above. When we assign entry to the last element.
        # Step 3 and 4.
        # XXX The specification says "and entry _not_ in" but that doesn't
        # work. This seems to give correct results too...
        while entry != Marker and entry in self.openElements:
            i -= 1
            entry = self.activeFormattingElements[i]
        while True:
            # XXX why clone?
            i += 1
            clone = self.activeFormattingElements[i].cloneNode()
            self.openElements[-1].appendChild(clone)
            self.openElements.append(clone)
            self.activeFormattingElements[i] = clone
            if clone == self.activeFormattingElements[-1]:
                break

    def clearActiveFormattingElements(self):
        entry = self.activeFormattingElements.pop()
        while self.activeFormattingElements and entry != Marker:
            entry = self.activeFormattingElements.pop()

    def elementInActiveFormattingElements(self, name):
        """Check if an element exists between the end of the active
        formatting elements and the last marker. If it does, return it, else
        return false"""

        for item in self.activeFormattingElements[::-1]:
            # Check for Marker first because if it's a Marker it doesn't have a
            # name attribute.
            if item == Marker:
                break
            elif item.name == name:
                return item
        return False

    def createElement(self, name, attributes):
        # XXX AT Change this if we ever implement different node types for
        # different elements
        element = Element(name)
        element.attributes = attributes
        return element

    def insertElement(self, name, attributes):
        element = self.createElement(name, attributes)
        if (not(self.insertFromTable) or (self.insertFromTable and 
                                          self.openElements[-1].name not in 
                                          tableInsertModeElements)):
            self.openElements[-1].appendChild(element)
            self.openElements.append(element)
        else:
            #We should be in the InTable mode. This means we want to do 
            #special magic element rearranging
            self.insertMisnestedNodeFromTable(element)
            self.openElements.append(element)

    def insertText(self, data, parent=None):
        node = TextNode(data)
        if parent is None: parent = self.openElements[-1]
        if (not(self.insertFromTable) or (self.insertFromTable and 
                                          self.openElements[-1].name not in 
                                          tableInsertModeElements)):
            parent.appendChild(node)
        else:
            #We should be in the InTable mode. This means we want to do 
            #special magic element rearranging
            self.insertMisnestedNodeFromTable(node)

    def insertMisnestedNodeFromTable(self, element):
        #The foster parent element is the one which comes before the most
        #recently opened table element
        #XXX - this is really inelegant
        lastTable = self.openElements[-1]
        for fosterParent in self.openElements[-2::-1]:
            if lastTable.name == u"table":
                break
            lastTable = fosterParent
        #If the last table element in the stack of open elements is a 
        #child of this foster parent element, then the new node must be 
        #inserted immediately before the last table element in the stack 
        #of open elements in this foster parent element
        if lastTable in fosterParent.childNodes:
            #XXX - do we need to change the stack of open elements here?
            fosterParent.insertBefore(element, lastTable)
        else:
            fosterParent.appendChild(element)

    def generateImpliedEndTags(self, exclude=None):
        name = self.openElements[-1].name
        if (name in frozenset(("dd", "dt", "li", "p", "td", "th", "tr"))
          and name != exclude):
            self.processEndTag(name)
            self.generateImpliedEndTags(exclude)

    def resetInsertionMode(self):
        last = False<commit_msg>Implement new version of InTable misnested element handling. Doesn't deal with text per testcases<commit_after>import tokenizer

from utils import utils
from constants import contentModelFlags, spaceCharacters
from constants import scopingElements, formattingElements, specialElements
from constants import headingElements, tableInsertModeElements
import inspect

"""The scope markers are inserted when entering buttons, object
elements, marquees, table cells, and table captions, and are used to
prevent formatting from "leaking" into tables, buttons, object
elements, and marquees."""
Marker = None

#Really crappy basic implementation of a DOM-core like thing
class Node(object):
    def __init__(self, name):
        self.name = name
        self.parent = None
        self.value = None
        self.childNodes = []
        self.attributes = {}
        self._flags = []

    def __str__(self):
        return self.name

    def __repr__(self):
        return "<%s %s>"%(self.__class__, self.name)

    def printTree(self, indent=0):
        tree = '\n|%s%s' % (' '*indent, str(self))
        for child in self.childNodes:
            tree += child.printTree(indent+2)
        return tree

    def appendChild(self, node, index=None):
        if (isinstance(node, TextNode) and self.childNodes and
          isinstance(self.childNodes[-1], TextNode)):
            self.childNodes[-1].value += node.value
        else:
            self.childNodes.append(node)
        node.parent = self

    def insertBefore(self, node, refNode):
        index = self.childNodes.index(refNode)
        if (isinstance(node, TextNode) and index > 0 and
          isinstance(self.childNodes[index - 1], TextNode)):
            self.childNodes[index - 1].value += node.value
        else:
            self.childNodes.insert(index, node)
        node.parent = self

    def removeChild(self, node):
        try:
            self.childNodes.remove(node)
        except:
            # XXX
            raise
        node.parent = None

    def cloneNode(self):
        newNode = type(self)(self.name)
        for attr, value in self.attributes.iteritems():
            newNode.attributes[attr] = value
        newNode.value = self.value
        return newNode

class Document(Node):
    def __init__(self):
        Node.__init__(self, None)

    def __str__(self):
        return '#document'

    def printTree(self):
        tree = str(self)
        for child in self.childNodes:
            tree += child.printTree(2)
        return tree

class DocumentType(Node):
    def __init__(self, name):
        Node.__init__(self, name)

    def __str__(self):
        return '<!DOCTYPE %s>' % self.name

class TextNode(Node):
    def __init__(self, value):
        Node.__init__(self, None)
        self.value = value

    def __str__(self):
        return '"%s"' % self.value

class Element(Node):
    def __init__(self, name):
        Node.__init__(self, name)

    def __str__(self):
        return '<%s>' % self.name

    def printTree(self, indent):
        tree = '\n|%s%s' % (' '*indent, str(self))
        indent += 2
        if self.attributes:
            for name, value in self.attributes.iteritems():
                tree += '\n|%s%s="%s"' % (' '*indent, name, value)
        for child in self.childNodes:
            tree += child.printTree(indent)
        return tree

class CommentNode(Node):
    def __init__(self, data):
        Node.__init__(self, None)
        self.data = data

    def __str__(self):
        return '<!-- %s -->' % self.data

class HTMLParser(object):
    """Main parser class"""

    def __init__(self, strict = False):
        # Raise an exception on the first error encountered
        self.strict = strict

        self.openElements = []
        self.activeFormattingElements = []
        self.headPointer = None
        self.formPointer = None

        self.phases = {"initial":InitialPhase,
                       "rootElement":RootElementPhase,
                       "main":MainPhase,
                       "trailingEnd":TrailingEndPhase}

    def parse(self, stream, innerHTML=False):
        """Stream should be a stream of unicode bytes. Character encoding
        issues have not yet been dealt with."""

        self.document = Document()

        #We don't actually support inner HTML yet but this should allow
        #assertations
        self.innerHTML = innerHTML

        #The parsing phase we are currently in
        self.phase = InitialPhase(self)

        #Flag indicationg special insertion mode from elements misnested inside
        #a table
        self.insertFromTable = False

        self.tokenizer = tokenizer.HTMLTokenizer(self)
        self.tokenizer.tokenize(stream)

        return self.document

    def processDoctype(self, name, error):
        self.phase.processDoctype(name, error)

    def processStartTag(self, name, attributes):
        self.phase.processStartTag(name, attributes)

    def processEndTag(self, name):
        self.phase.processEndTag(name)

    def processComment(self, data):
        self.phase.processComment(data)

    def processCharacter(self, data):
        self.phase.processCharacter(data)

    def processEOF(self):
        self.phase.processEOF()

    def parseError(self):
        if self.strict:
            raise ParseError

    def atheistParseError(self):
        """This error is not an error"""
        pass

    def switchPhase(self, name):
        """Switch between different phases of the parsing
        """
        # Need to hang on to state between trailing end phase and main phase
        if (name == "trailingEnd" and
            isinstance(self.phase, self.phases["main"])):
            self.mainPhaseState = self.phase
            self.phase = self.phases["trailingEnd"](self)
        elif (name == "main" and
              isinstance(self.phase, self.phases["trailingEnd"])):
            self.phase = self.mainPhaseState
        else:
            self.phase = self.phases[name](self)

    #XXX - almost everthing after this point should be moved into a
    #seperate treebuilder object

    def switchInsertionMode(self, name):
        """Switch between different insertion modes in the main phase"""
        # XXX AT Arguably this should be on the main phase object itself
        self.phase.insertionMode = self.phase.insertionModes[name](self)

    def elementInScope(self, target, tableVariant=False):
        # AT Use reverse instead of [::-1] when we can rely on Python 2.4
        # AT How about while True and simply set node to [-1] and set it to
        # [-2] at the end...
        for node in self.openElements[::-1]:
            if node.name == target:
                return True
            elif node.name == "table":
                return False
            elif not tableVariant and node.name in scopingElements:
                return False
            elif node.name == "html":
                return False
        assert False # We should never reach this point

    def reconstructActiveFormattingElements(self):
        # This covers the ", if any" case mentioned often in the drafts, before
        # you have to reconstruct the active formatting elements.
        if not self.activeFormattingElements:
            return

        # We start with the last element. So i is -1.
        i = -1
        entry = self.activeFormattingElements[i]

        # Step 1
        if entry == Marker or entry in self.openElements:
            return

        # Step 2 is covered above. When we assign entry to the last element.
        # Step 3 and 4.
        # XXX The specification says "and entry _not_ in" but that doesn't
        # work. This seems to give correct results too...
        while entry != Marker and entry in self.openElements:
            i -= 1
            entry = self.activeFormattingElements[i]
        while True:
            # XXX why clone?
            i += 1
            clone = self.activeFormattingElements[i].cloneNode()
            self.openElements[-1].appendChild(clone)
            self.openElements.append(clone)
            self.activeFormattingElements[i] = clone
            if clone == self.activeFormattingElements[-1]:
                break

    def clearActiveFormattingElements(self):
        entry = self.activeFormattingElements.pop()
        while self.activeFormattingElements and entry != Marker:
            entry = self.activeFormattingElements.pop()

    def elementInActiveFormattingElements(self, name):
        """Check if an element exists between the end of the active
        formatting elements and the last marker. If it does, return it, else
        return false"""

        for item in self.activeFormattingElements[::-1]:
            # Check for Marker first because if it's a Marker it doesn't have a
            # name attribute.
            if item == Marker:
                break
            elif item.name == name:
                return item
        return False

    def createElement(self, name, attributes):
        # XXX AT Change this if we ever implement different node types for
        # different elements
        element = Element(name)
        element.attributes = attributes
        return element

    def insertElement(self, name, attributes):
        element = self.createElement(name, attributes)
        if (not(self.insertFromTable) or (self.insertFromTable and 
                                          self.openElements[-1].name not in 
                                          tableInsertModeElements)):
            self.openElements[-1].appendChild(element)
            self.openElements.append(element)
        else:
            #We should be in the InTable mode. This means we want to do 
            #special magic element rearranging
            self.insertMisnestedNodeFromTable(element)
            self.openElements.append(element)

    def insertText(self, data, parent=None):
        node = TextNode(data)
        if parent is None: 
            parent = self.openElements[-1]
        if (not(self.insertFromTable) or (self.insertFromTable and 
                                          self.openElements[-1].name not in 
                                          tableInsertModeElements)):
            parent.appendChild(node)
        else:
            #We should be in the InTable mode. This means we want to do 
            #special magic element rearranging
            self.insertMisnestedNodeFromTable(node)

    def insertMisnestedNodeFromTable(self, element):
        #The foster parent element is the one which comes before the most
        #recently opened table element
        #XXX - this is really inelegant
        lastTable=None
        for elm in self.openElements[::-1]:
            if elm.name == u"table":
                lastTable = elm
                break
        if lastTable:
            #XXX - we should really check that this parent is actually a 
            #node here
            if lastTable.parent:
                fosterParent = lastTable.parent
                fosterParent.insertBefore(element, lastTable)
            else:
                fosterParent = self.openElements[
                    self.openElements.index(lastTable) - 1]
                fosterParent.appendChild(element)
        else:
            assert self.innerHTML
            fosterParent = self.openElements[0]
            fosterParent.appendChild(element)
        import sys
        sys.stderr.write(repr(fosterParent))

    def generateImpliedEndTags(self, exclude=None):
        name = self.openElements[-1].name
        if (name in frozenset(("dd", "dt", "li", "p", "td", "th", "tr"))
          and name != exclude):
            self.processEndTag(name)
            self.generateImpliedEndTags(exclude)

    def resetInsertionMode(self):
        last = False
{'fraction_non_alphanumeric': 0.08734372324028088, 'fraction_numerical': 0.002868641890734715, 'mean_word_length': 2.0846539883782356, 'pattern_counts': {'":': 14, '<': 11, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '438690', 'n_tokens_mistral': 6119, 'n_tokens_neox': 5894, 'n_words': 1935}