compare two html documents import html from html parser import htmlparser from django utils html import voidelements from django utils regexhelper import lazyrecompile ascii whitespace is u0009 tab u000a lf u000c ff u000d cr or u0020 space https infra spec whatwg orgasciiwhitespace asciiwhitespace lazyrecompilertnfr https html spec whatwg orgattributes3 booleanattributes allowfullscreen async autofocus autoplay checked controls default defer disabled formnovalidate hidden ismap itemscope loop multiple muted nomodule novalidate open playsinline readonly required reversed selected attributes for deprecated tags truespeed def normalizewhitespacestring return asciiwhitespace sub string def normalizeattributesattributes normalized for name value in attributes if name class and value special case handling of class attribute so that comparisons of dom instances are not sensitive to ordering of classes value join sortedvalue for value in asciiwhitespace splitvalue if value boolean attributes without a value is same as attribute with value that equals the attributes name for example input checked input checkedchecked if name in booleanattributes if not value or value name value none elif value is none value normalized appendname value return normalized class element def initself name attributes self name name self attributes sortedattributes self children def appendself element if isinstanceelement str element normalizewhitespaceelement if self children and isinstanceself children1 str self children1 element self children1 normalizewhitespaceself children1 return elif self children removing last children if it is only whitespace this can result in incorrect dom representations since whitespace between inline tags like span is significant if isinstanceself children1 str and self children1 isspace self children pop if element self children appendelement def finalizeself def rstriplastelementchildren if children and isinstancechildren1 str children1 children1 rstrip if not children1 children pop children rstriplastelementchildren return children rstriplastelementself children for i child in enumerateself children if isinstancechild str self childreni child strip elif hasattrchild finalize child finalize def eqself element if not hasattrelement name or self name element name return false if self attributes element attributes return false return self children element children def hashself return hashself name self attributes def countself element counttrue if not isinstanceelement str and self element return 1 if isinstanceelement rootelement and self children element children return 1 i 0 elemchildidx 0 for child in self children child is text content and element is also text content then make a simple text in text if isinstancechild str if isinstanceelement str if count i child countelement elif element in child return 1 else look for element wholly within this child i child countelement countcount if not count and i return i also look for a sequence of element s children among self s children self children element children is tested above but will fail if self has additional children ex ab is contained in abc if isinstanceelement rootelement and element children elemchild element childrenelemchildidx start or continue match advance index if elemchild child elemchildidx 1 match found reset index if elemchildidx lenelement children i 1 elemchildidx 0 no match reset index else elemchildidx 0 return i def containsself element return self countelement countfalse 0 def countself element return self countelement counttrue def getitemself key return self childrenkey def strself output s self name for key value in self attributes if value is not none output ss key value else output s key if self children output n output join html escapec if isinstancec str else strc for c in self children output ns self name else output return output def reprself return strself class rootelementelement def initself super initnone def strself return join html escapec if isinstancec str else strc for c in self children class htmlparseerrorexception pass class parserhtmlparser def initself super init self root rootelement self opentags self elementpositions def errorself msg raise htmlparseerrormsg self getpos def formatpositionself positionnone elementnone if not position and element position self elementpositionselement if position is none position self getpos if hasattrposition lineno position position lineno position offset return line d column d position property def currentself if self opentags return self opentags1 else return self root def handlestartendtagself tag attrs self handlestarttagtag attrs if tag not in voidelements self handleendtagtag def handlestarttagself tag attrs attrs normalizeattributesattrs element elementtag attrs self current appendelement if tag not in voidelements self opentags appendelement self elementpositionselement self getpos def handleendtagself tag if not self opentags self errorunexpected end tag s s tag self formatposition element self opentags pop while element name tag if not self opentags self error unexpected end tag s s tag self formatposition element self opentags pop def handledataself data self current appenddata def parsehtmlhtml parser parser parser feedhtml parser close document parser root document finalize removing root element if it s not necessary if lendocument children 1 and not isinstancedocument children0 str document document children0 return document ascii whitespace is u 0009 tab u 000a lf u 000c ff u 000d cr or u 0020 space https infra spec whatwg org ascii whitespace https html spec whatwg org attributes 3 attributes for deprecated tags special case handling of class attribute so that comparisons of dom instances are not sensitive to ordering of classes boolean attributes without a value is same as attribute with value that equals the attributes name for example input checked input checked checked removing last children if it is only whitespace this can result in incorrect dom representations since whitespace between inline tags like span is significant child is text content and element is also text content then make a simple text in text look for element wholly within this child also look for a sequence of element s children among self s children self children element children is tested above but will fail if self has additional children ex a b is contained in a b c start or continue match advance index match found reset index no match reset index take a string that contains html and turn it into a python object structure that can be easily compared against other html on semantic equivalence syntactical differences like which quotation is used on arguments will be ignored removing root element if it s not necessary
import html
from html.parser import HTMLParser

from django.utils.html import VOID_ELEMENTS
from django.utils.regex_helper import _lazy_re_compile




ASCII_WHITESPACE = _lazy_re_compile(r"[\t\n\f\r ]+")


BOOLEAN_ATTRIBUTES = {
    "allowfullscreen",
    "async",
    "autofocus",
    "autoplay",
    "checked",
    "controls",
    "default",
    "defer ",
    "disabled",
    "formnovalidate",
    "hidden",
    "ismap",
    "itemscope",
    "loop",
    "multiple",
    "muted",
    "nomodule",
    "novalidate",
    "open",
    "playsinline",
    "readonly",
    "required",
    "reversed",
    "selected",
    
    "truespeed",
}


def normalize_whitespace(string):
    return ASCII_WHITESPACE.sub(" ", string)


def normalize_attributes(attributes):
    normalized = []
    for name, value in attributes:
        if name == "class" and value:
            
            
            value = " ".join(
                sorted(value for value in ASCII_WHITESPACE.split(value) if value)
            )
        
        
        
        if name in BOOLEAN_ATTRIBUTES:
            if not value or value == name:
                value = None
        elif value is None:
            value = ""
        normalized.append((name, value))
    return normalized


class Element:
    def __init__(self, name, attributes):
        self.name = name
        self.attributes = sorted(attributes)
        self.children = []

    def append(self, element):
        if isinstance(element, str):
            element = normalize_whitespace(element)
            if self.children and isinstance(self.children[-1], str):
                self.children[-1] += element
                self.children[-1] = normalize_whitespace(self.children[-1])
                return
        elif self.children:
            
            
            
            if isinstance(self.children[-1], str) and self.children[-1].isspace():
                self.children.pop()
        if element:
            self.children.append(element)

    def finalize(self):
        def rstrip_last_element(children):
            if children and isinstance(children[-1], str):
                children[-1] = children[-1].rstrip()
                if not children[-1]:
                    children.pop()
                    children = rstrip_last_element(children)
            return children

        rstrip_last_element(self.children)
        for i, child in enumerate(self.children):
            if isinstance(child, str):
                self.children[i] = child.strip()
            elif hasattr(child, "finalize"):
                child.finalize()

    def __eq__(self, element):
        if not hasattr(element, "name") or self.name != element.name:
            return False
        if self.attributes != element.attributes:
            return False
        return self.children == element.children

    def __hash__(self):
        return hash((self.name, *self.attributes))

    def _count(self, element, count=True):
        if not isinstance(element, str) and self == element:
            return 1
        if isinstance(element, RootElement) and self.children == element.children:
            return 1
        i = 0
        elem_child_idx = 0
        for child in self.children:
            
            
            if isinstance(child, str):
                if isinstance(element, str):
                    if count:
                        i += child.count(element)
                    elif element in child:
                        return 1
            else:
                
                i += child._count(element, count=count)
                if not count and i:
                    return i
                
                
                
                
                if isinstance(element, RootElement) and element.children:
                    elem_child = element.children[elem_child_idx]
                    
                    if elem_child == child:
                        elem_child_idx += 1
                        
                        if elem_child_idx == len(element.children):
                            i += 1
                            elem_child_idx = 0
                    
                    else:
                        elem_child_idx = 0
        return i

    def __contains__(self, element):
        return self._count(element, count=False) > 0

    def count(self, element):
        return self._count(element, count=True)

    def __getitem__(self, key):
        return self.children[key]

    def __str__(self):
        output = "<%s" % self.name
        for key, value in self.attributes:
            if value is not None:
                output += ' %s="%s"' % (key, value)
            else:
                output += " %s" % key
        if self.children:
            output += ">\n"
            output += "".join(
                [
                    html.escape(c) if isinstance(c, str) else str(c)
                    for c in self.children
                ]
            )
            output += "\n</%s>" % self.name
        else:
            output += ">"
        return output

    def __repr__(self):
        return str(self)


class RootElement(Element):
    def __init__(self):
        super().__init__(None, ())

    def __str__(self):
        return "".join(
            [html.escape(c) if isinstance(c, str) else str(c) for c in self.children]
        )


class HTMLParseError(Exception):
    pass


class Parser(HTMLParser):
    def __init__(self):
        super().__init__()
        self.root = RootElement()
        self.open_tags = []
        self.element_positions = {}

    def error(self, msg):
        raise HTMLParseError(msg, self.getpos())

    def format_position(self, position=None, element=None):
        if not position and element:
            position = self.element_positions[element]
        if position is None:
            position = self.getpos()
        if hasattr(position, "lineno"):
            position = position.lineno, position.offset
        return "Line %d, Column %d" % position

    @property
    def current(self):
        if self.open_tags:
            return self.open_tags[-1]
        else:
            return self.root

    def handle_startendtag(self, tag, attrs):
        self.handle_starttag(tag, attrs)
        if tag not in VOID_ELEMENTS:
            self.handle_endtag(tag)

    def handle_starttag(self, tag, attrs):
        attrs = normalize_attributes(attrs)
        element = Element(tag, attrs)
        self.current.append(element)
        if tag not in VOID_ELEMENTS:
            self.open_tags.append(element)
        self.element_positions[element] = self.getpos()

    def handle_endtag(self, tag):
        if not self.open_tags:
            self.error("Unexpected end tag `%s` (%s)" % (tag, self.format_position()))
        element = self.open_tags.pop()
        while element.name != tag:
            if not self.open_tags:
                self.error(
                    "Unexpected end tag `%s` (%s)" % (tag, self.format_position())
                )
            element = self.open_tags.pop()

    def handle_data(self, data):
        self.current.append(data)


def parse_html(html):
    
    parser = Parser()
    parser.feed(html)
    parser.close()
    document = parser.root
    document.finalize()
    
    if len(document.children) == 1 and not isinstance(document.children[0], str):
        document = document.children[0]
    return document