<commit_before>import re
import sys
import types
try:
    from HTMLParser import HTMLParser
except ImportError:  # pragma: no cover
    from html.parser import HTMLParser

import validator.testcases.scripting as scripting
import validator.unicodehelper as unicodehelper
from validator.testcases.markup import csstester
from validator.contextgenerator import ContextGenerator
from validator.constants import *
from validator.decorator import versions_after

DEBUG = False

UNSAFE_TAGS = ("script", "object", "embed", "base", )
UNSAFE_THEME_TAGS = ("implementation", "browser", "xul:browser", "xul:script")
SELF_CLOSING_TAGS = ("area", "base", "basefont", "br", "col", "frame", "hr",
                     "img", "input", "li", "link", "meta", "p", "param", )
SAFE_IFRAME_TYPES = ("content", "content-primary", "content-targetable", )
TAG_NOT_OPENED = "Tag (%s) being closed before it is opened."

DOM_MUTATION_HANDLERS = (
        "ondomattrmodified", "ondomattributenamechanged",
        "ondomcharacterdatamodified", "ondomelementnamechanged",
        "ondomnodeinserted", "ondomnodeinsertedintodocument", "ondomnoderemoved",
        "ondomnoderemovedfromdocument", "ondomsubtreemodified", )
UNSAFE_THEME_XBL = ("constructor", "destructor", "field", "getter",
                    "implementation", "setter", )


class MarkupParser(HTMLParser):
    """Parse and analyze the versious components of markup files."""

    def __init__(self, err, strict=True, debug=False):
        HTMLParser.__init__(self)
        self.err = err
        self.line = 0
        self.strict = strict
        self.debug = debug

        self.context = None

        self.xml_state = []
        self.xml_line_stack = []
        self.xml_buffer = []
        self.xbl = False

        self.reported = set()

    def process(self, filename, data, extension="xul"):
        """Processes data by splitting it into individual lines, then
        incrementally feeding each line into the parser, increasing the
        value of the line number with each line."""

        self.line = 0
        self.filename = filename
        self.extension = extension.lower()

        self.reported = set()

        self.context = ContextGenerator(data)

        lines = data.split("\n")
        line_buffer = []
        force_buffer = False
        pline = 0
        for line in lines:
            self.line += 1

            search_line = line
            while True:
                # CDATA elements are gross. Pass the whole entity as one chunk
                if "<![CDATA[" in search_line and not force_buffer:
                    cdatapos = search_line.find("<![CDATA[")
                    post_cdata = search_line[cdatapos:]

                    if "]]>" in post_cdata:
                        search_line = post_cdata[post_cdata.find("]]>") + 3:]
                        continue
                    force_buffer = True
                elif "]]>" in search_line and force_buffer:
                    force_buffer = False
                break

            if force_buffer:
                line_buffer.append(line)
            else:
                if line_buffer:
                    line_buffer.append(line)
                    line = "\n".join(line_buffer)
                    line_buffer = []
                self._feed_parser(line)

    def _feed_parser(self, line):
        "Feeds data into the parser"

        line = unicodehelper.decode(line)

        try:
            self.feed(line + "\n")
        except UnicodeDecodeError, exc_instance:
            exc_class, val, traceback = sys.exc_info()
            try:
                line = line.decode("ascii", "ignore")
                self.feed(line + "\n")
            except:
                raise exc_instance, None, traceback

        except Exception as inst:
            if DEBUG:  # pragma: no cover
                print self.xml_state, inst

            if "markup" in self.reported:
                return

            if ("script" in self.xml_state or
                self.debug and "testscript" in self.xml_state):
                if "script_comments" in self.reported or not self.strict:
                    return
                self.err.notice(("testcases_markup_markuptester",
                                 "_feed",
                                 "missing_script_comments"),
                                "Missing comments in <script> tag",
                                "Markup parsing errors occurred while trying "
                                "to parse the file. This would likely be "
                                "mitigated by wrapping <script> tag contents "
                                "in HTML comment tags (<!-- -->)",
                                self.filename,
                                line=self.line,
                                context=self.context,
                                tier=2)
                self.reported.add("script_comments")
                return

            if self.strict:
                self.err.warning(("testcases_markup_markuptester",
                                  "_feed",
                                  "parse_error"),
                                 "Markup parsing error",
                                 ["There was an error parsing a markup "
                                  "file.",
                                  str(inst)],
                                 self.filename,
                                 line=self.line,
                                 context=self.context)
            self.reported.add("markup")

    def handle_startendtag(self, tag, attrs):
        self.handle_starttag(tag, attrs, True)
        self.handle_endtag(tag)

    def handle_starttag(self, tag, attrs, self_closing=False):

        # Normalize!
        tag = tag.lower()
        # XUL scripts are identical to normal scripts. Treat them the same.
        if tag == "xul:script":
            tag = "script"

        orig_tag = tag

        # Be extra sure it's not a self-closing tag.
        if not self_closing:
            self_closing = tag in SELF_CLOSING_TAGS

        if DEBUG:  # pragma: no cover
            print "S: ", self.xml_state, tag, self_closing

        # A fictional tag for testing purposes.
        if tag == "xbannedxtestx":
            self.err.error(("testcases_markup_markuptester",
                            "handle_starttag",
                            "banned_element"),
                           "Banned element",
                           "A banned element was detected",
                           self.filename,
                           line=self.line,
                           context=self.context)

        # Test for banned XBL in themes.
        if self.err.detected_type == PACKAGE_THEME:
            if tag.startswith("xbl:"):
                self.xbl = True
                tag = tag[4:]

            # Find XBL elements.
            if self.xbl:
                if tag in UNSAFE_THEME_XBL:
                    self.err.warning(
                        err_id=("testcases_markup_markuptester",
                                "handle_starttag",
                                "unsafe_theme_xbl_element"),
                        warning="Banned XBL element in theme.",
                        description=["Certain XBL elements are disallowed in "
                                     "themes.",
                                     "Element: <xbl:%s>" % tag],
                        filename=self.filename,
                        line=self.line,
                        context=self.context)

                elif (tag == "property" and
                      any(a[0] in (u"onset", u"onget") for a in attrs)):
                    self.err.warning(
                        err_id=("testcases_markup_markuptester",
                                "handle_starttag",
                                "theme_xbl_property"),
                        warning="Themes are not allowed to use XBL properties",
                        description="XBL properties cannot be used in themes.",
                        filename=self.filename,
                        line=self.line,
                        context=self.context)


        # Test for banned elements in language pack and theme markup.
        if self.err.detected_type in (PACKAGE_LANGPACK, PACKAGE_THEME):
            if (tag in UNSAFE_TAGS or
                (self.err.detected_type == PACKAGE_THEME and
                 tag in UNSAFE_THEME_TAGS)):
                self.err.warning(("testcases_markup_markuptester",
                                  "handle_starttag",
                                  "unsafe_langpack_theme"),
                                 "Unsafe tag for add-on type",
                                 ["A tag in your markup has been marked as "
                                  "being potentially unsafe. Consider "
                                  "alternate means of accomplishing what the "
                                  "code executed by this tag performs.",
                                  'Tag "%s" is disallowed.' % tag],
                                 self.filename,
                                 line=self.line,
                                 context=self.context)
                if DEBUG:  # pragma: no cover
                    print "Unsafe Tag ------"

            # Make sure all src/href attributes are local
            for attr in attrs:
                if attr[0].lower() in ("src", "href") and \
                   not self._is_url_local(attr[1].lower()):
                    self.err.warning(("testcases_markup_markuptester",
                                      "handle_starttag",
                                      "remote_src_href"),
                                     "src/href attributes must be local.",
                                     "Language packs require that all src and "
                                     "href attributes are relative URLs.",
                                     self.filename,
                                     line=self.line,
                                     context=self.context)
                    self.err.reject = True

        if tag in ("iframe", "browser") and self.extension == "xul":
            # Bork if XUL iframe has no type attribute

            type_ = None
            src = None
            for attr in attrs:
                attr_name = attr[0].lower()
                if attr_name == "type":
                    type_ = attr[1].lower()
                elif attr_name == "src":
                    src = attr[1].lower()

            # We say it's true by default to catch elements that are
            # type="chrome" without an src="" attribute.
            remote_src = True
            if isinstance(src, types.StringTypes):
                remote_src = not self._is_url_local(src)

            if (type_ and
                not (type_ in SAFE_IFRAME_TYPES or
                     not remote_src)):
                self.err.warning(("testcases_markup_markuptester",
                                  "handle_starttag",
                                  "iframe_type_unsafe"),
                                 "iframe/browser missing 'type' attribute",
                                 "All iframe and browser elements must have "
                                 "either a valid `type` attribute or a `src` "
                                 "attribute that points to a local file.",
                                 self.filename,
                                 line=self.line,
                                 context=self.context)
            elif ((not type_ or
                   type_ not in SAFE_IFRAME_TYPES) and
                  remote_src):
                self.err.warning(("testcases_markup_markuptester",
                                  "handle_starttag",
                                  "iframe_type_unsafe"),
                                 "Typeless iframes/browsers must be local.",
                                 "iframe and browser elements that lack a type "
                                 "attribute must always have src attributes "
                                 "that reference local resources.",
                                 self.filename,
                                 line=self.line,
                                 context=self.context)

        elif tag == "script" and self.extension == "xul":
            # Per the Addon Validator Spec (v2), scripts in XUL
            # must not be remote.

            src = None
            for attr in attrs:
                if attr[0].lower() == "src":
                    src = attr[1].lower()

            if src and not self._is_url_local(src):
                self.err.warning(("testcases_markup_markuptester",
                                  "handle_starttag",
                                  "banned_remote_scripts"),
                                 "Scripts must not be remote in XUL",
                                 "In XUL, <script> tags must not be referenced "
                                 "to script files that are hosted remotely.",
                                 self.filename,
                                 line=self.line,
                                 context=self.context)

        # Find CSS and JS attributes and handle their values like they
        # would otherwise be handled by the standard parser flow.
        for attr in attrs:
            attr_name, attr_value = attr[0].lower(), attr[1]

            if (attr_name == "xmlns:xbl" and
                attr_value == "http://www.mozilla.org/xbl"):
                self.xbl = True

            if (self.err.detected_type == PACKAGE_THEME and
                attr_value.startswith(("data:", "javascript:"))):

                self.err.warning(
                        err_id=("testcases_markup_markuptester",
                                "handle_starttag",
                                "theme_attr_prefix"),
                        warning="Attribute contains banned prefix",
                        description=["A mark element's attribute contains a "
                                     "prefix which is not allowed in themes.",
                                     "Attribute: %s" % attr_name],
                        filename=self.filename,
                        line=self.line,
                        context=self.context)


            if attr_name == "style":
                csstester.test_css_snippet(self.err,
                                           self.filename,
                                           attr_value,
                                           self.line)
            elif attr_name.startswith("on"):  # JS attribute
                # Warn about DOM mutation event handlers.
                if attr_name in DOM_MUTATION_HANDLERS:
                    self.err.warning(
                        err_id=("testcases_markup_markuptester",
                                "handle_starttag",
                                "dom_manipulation_handler"),
                        warning="DOM Mutation Events Prohibited",
                        description="DOM mutation events are flagged because "
                                    "of their deprecated status, as well as "
                                    "their extreme inefficiency. Consider "
                                    "using a different event.",
                        filename=self.filename,
                        line=self.line,
                        context=self.context)

                scripting.test_js_snippet(err=self.err,
                                          data=attr_value,
                                          filename=self.filename,
<<<<<<< HEAD
                                          line=self.line)
=======
                                          line=self.line,
                                          context=self.context)
>>>>>>> 292f7d0060bb6b60a68c415fbef29562f8a41a3e

            elif (self.extension == "xul" and
                  attr_name in ("insertbefore", "insertafter") and
                  any((id in attr[1]) for id in ("menu_pageSource",
                                                 "menu_pageinspect",
                                                 "javascriptConsole",
                                                 "webConsole"))):
                self.err.notice(
                    err_id=("testcases_markup_markuptester",
                            "handle_starttag",
                            "incompatible_menu_items"),
                    notice="Menu item has been moved",
                    description="Your add-on has an overlay that uses the "
                                "insertbefore or insertafter attribute "
                                "pointing to menuitems that have been moved "
                                "to a different menu item. Your overlay items "
                                "may appear in unexpected locations because "
                                "of this. See "
                        "https://bugzilla.mozilla.org/show_bug.cgi?id=653221"
                                " for more information.",
                    filename=self.filename,
                    line=self.line,
                    context=self.context,
                    for_appversions={'{ec8030f7-c20a-464f-9b0e-13a3a9e97384}':
                                         versions_after("firefox", "6.0a1")},
                    compatibility_type="warning")


        # When the dev forgets their <!-- --> on a script tag, bad
        # things happen.
        if "script" in self.xml_state and tag != "script":
            self._save_to_buffer("<" + tag + self._format_args(attrs) + ">")
            return

        self.xml_state.append(orig_tag)
<<<<<<< HEAD
=======
        self.xml_line_stack.append(self.line)
>>>>>>> 292f7d0060bb6b60a68c415fbef29562f8a41a3e
        self.xml_buffer.append(unicode(""))

    def handle_endtag(self, tag):

        tag = tag.lower()
        if tag == "xul:script":
            tag = "script"

        if DEBUG:  # pragma: no cover
            print "E: ", tag, self.xml_state

        if not self.xml_state:
            if "closing_tags" in self.reported or not self.strict:
                if DEBUG:
                    print "Unstrict; extra closing tags ------"
                return
            self.err.warning(("testcases_markup_markuptester",
                              "handle_endtag",
                              "extra_closing_tags"),
                             "Markup parsing error",
                             "The markup file has more closing tags than it "
                             "has opening tags.",
                             self.filename,
                             line=self.line,
                             context=self.context,
                             tier=2)
            self.reported.add("closing_tags")
            if DEBUG:  # pragma: no cover
                print "Too many closing tags ------"
            return
<commit_msg>Fixed bad merge. Why does vim hate me?<commit_after>import re
import sys
import types
try:
    from HTMLParser import HTMLParser
except ImportError:  # pragma: no cover
    from html.parser import HTMLParser

import validator.testcases.scripting as scripting
import validator.unicodehelper as unicodehelper
from validator.testcases.markup import csstester
from validator.contextgenerator import ContextGenerator
from validator.constants import *
from validator.decorator import versions_after

DEBUG = False

UNSAFE_TAGS = ("script", "object", "embed", "base", )
UNSAFE_THEME_TAGS = ("implementation", "browser", "xul:browser", "xul:script")
SELF_CLOSING_TAGS = ("area", "base", "basefont", "br", "col", "frame", "hr",
                     "img", "input", "li", "link", "meta", "p", "param", )
SAFE_IFRAME_TYPES = ("content", "content-primary", "content-targetable", )
TAG_NOT_OPENED = "Tag (%s) being closed before it is opened."

DOM_MUTATION_HANDLERS = (
        "ondomattrmodified", "ondomattributenamechanged",
        "ondomcharacterdatamodified", "ondomelementnamechanged",
        "ondomnodeinserted", "ondomnodeinsertedintodocument", "ondomnoderemoved",
        "ondomnoderemovedfromdocument", "ondomsubtreemodified", )
UNSAFE_THEME_XBL = ("constructor", "destructor", "field", "getter",
                    "implementation", "setter", )


class MarkupParser(HTMLParser):
    """Parse and analyze the versious components of markup files."""

    def __init__(self, err, strict=True, debug=False):
        HTMLParser.__init__(self)
        self.err = err
        self.line = 0
        self.strict = strict
        self.debug = debug

        self.context = None

        self.xml_state = []
        self.xml_line_stack = []
        self.xml_buffer = []
        self.xbl = False

        self.reported = set()

    def process(self, filename, data, extension="xul"):
        """Processes data by splitting it into individual lines, then
        incrementally feeding each line into the parser, increasing the
        value of the line number with each line."""

        self.line = 0
        self.filename = filename
        self.extension = extension.lower()

        self.reported = set()

        self.context = ContextGenerator(data)

        lines = data.split("\n")
        line_buffer = []
        force_buffer = False
        pline = 0
        for line in lines:
            self.line += 1

            search_line = line
            while True:
                # CDATA elements are gross. Pass the whole entity as one chunk
                if "<![CDATA[" in search_line and not force_buffer:
                    cdatapos = search_line.find("<![CDATA[")
                    post_cdata = search_line[cdatapos:]

                    if "]]>" in post_cdata:
                        search_line = post_cdata[post_cdata.find("]]>") + 3:]
                        continue
                    force_buffer = True
                elif "]]>" in search_line and force_buffer:
                    force_buffer = False
                break

            if force_buffer:
                line_buffer.append(line)
            else:
                if line_buffer:
                    line_buffer.append(line)
                    line = "\n".join(line_buffer)
                    line_buffer = []
                self._feed_parser(line)

    def _feed_parser(self, line):
        "Feeds data into the parser"

        line = unicodehelper.decode(line)

        try:
            self.feed(line + "\n")
        except UnicodeDecodeError, exc_instance:
            exc_class, val, traceback = sys.exc_info()
            try:
                line = line.decode("ascii", "ignore")
                self.feed(line + "\n")
            except:
                raise exc_instance, None, traceback

        except Exception as inst:
            if DEBUG:  # pragma: no cover
                print self.xml_state, inst

            if "markup" in self.reported:
                return

            if ("script" in self.xml_state or
                self.debug and "testscript" in self.xml_state):
                if "script_comments" in self.reported or not self.strict:
                    return
                self.err.notice(("testcases_markup_markuptester",
                                 "_feed",
                                 "missing_script_comments"),
                                "Missing comments in <script> tag",
                                "Markup parsing errors occurred while trying "
                                "to parse the file. This would likely be "
                                "mitigated by wrapping <script> tag contents "
                                "in HTML comment tags (<!-- -->)",
                                self.filename,
                                line=self.line,
                                context=self.context,
                                tier=2)
                self.reported.add("script_comments")
                return

            if self.strict:
                self.err.warning(("testcases_markup_markuptester",
                                  "_feed",
                                  "parse_error"),
                                 "Markup parsing error",
                                 ["There was an error parsing a markup "
                                  "file.",
                                  str(inst)],
                                 self.filename,
                                 line=self.line,
                                 context=self.context)
            self.reported.add("markup")

    def handle_startendtag(self, tag, attrs):
        self.handle_starttag(tag, attrs, True)
        self.handle_endtag(tag)

    def handle_starttag(self, tag, attrs, self_closing=False):

        # Normalize!
        tag = tag.lower()
        # XUL scripts are identical to normal scripts. Treat them the same.
        if tag == "xul:script":
            tag = "script"

        orig_tag = tag

        # Be extra sure it's not a self-closing tag.
        if not self_closing:
            self_closing = tag in SELF_CLOSING_TAGS

        if DEBUG:  # pragma: no cover
            print "S: ", self.xml_state, tag, self_closing

        # A fictional tag for testing purposes.
        if tag == "xbannedxtestx":
            self.err.error(("testcases_markup_markuptester",
                            "handle_starttag",
                            "banned_element"),
                           "Banned element",
                           "A banned element was detected",
                           self.filename,
                           line=self.line,
                           context=self.context)

        # Test for banned XBL in themes.
        if self.err.detected_type == PACKAGE_THEME:
            if tag.startswith("xbl:"):
                self.xbl = True
                tag = tag[4:]

            # Find XBL elements.
            if self.xbl:
                if tag in UNSAFE_THEME_XBL:
                    self.err.warning(
                        err_id=("testcases_markup_markuptester",
                                "handle_starttag",
                                "unsafe_theme_xbl_element"),
                        warning="Banned XBL element in theme.",
                        description=["Certain XBL elements are disallowed in "
                                     "themes.",
                                     "Element: <xbl:%s>" % tag],
                        filename=self.filename,
                        line=self.line,
                        context=self.context)

                elif (tag == "property" and
                      any(a[0] in (u"onset", u"onget") for a in attrs)):
                    self.err.warning(
                        err_id=("testcases_markup_markuptester",
                                "handle_starttag",
                                "theme_xbl_property"),
                        warning="Themes are not allowed to use XBL properties",
                        description="XBL properties cannot be used in themes.",
                        filename=self.filename,
                        line=self.line,
                        context=self.context)


        # Test for banned elements in language pack and theme markup.
        if self.err.detected_type in (PACKAGE_LANGPACK, PACKAGE_THEME):
            if (tag in UNSAFE_TAGS or
                (self.err.detected_type == PACKAGE_THEME and
                 tag in UNSAFE_THEME_TAGS)):
                self.err.warning(("testcases_markup_markuptester",
                                  "handle_starttag",
                                  "unsafe_langpack_theme"),
                                 "Unsafe tag for add-on type",
                                 ["A tag in your markup has been marked as "
                                  "being potentially unsafe. Consider "
                                  "alternate means of accomplishing what the "
                                  "code executed by this tag performs.",
                                  'Tag "%s" is disallowed.' % tag],
                                 self.filename,
                                 line=self.line,
                                 context=self.context)
                if DEBUG:  # pragma: no cover
                    print "Unsafe Tag ------"

            # Make sure all src/href attributes are local
            for attr in attrs:
                if attr[0].lower() in ("src", "href") and \
                   not self._is_url_local(attr[1].lower()):
                    self.err.warning(("testcases_markup_markuptester",
                                      "handle_starttag",
                                      "remote_src_href"),
                                     "src/href attributes must be local.",
                                     "Language packs require that all src and "
                                     "href attributes are relative URLs.",
                                     self.filename,
                                     line=self.line,
                                     context=self.context)
                    self.err.reject = True

        if tag in ("iframe", "browser") and self.extension == "xul":
            # Bork if XUL iframe has no type attribute

            type_ = None
            src = None
            for attr in attrs:
                attr_name = attr[0].lower()
                if attr_name == "type":
                    type_ = attr[1].lower()
                elif attr_name == "src":
                    src = attr[1].lower()

            # We say it's true by default to catch elements that are
            # type="chrome" without an src="" attribute.
            remote_src = True
            if isinstance(src, types.StringTypes):
                remote_src = not self._is_url_local(src)

            if (type_ and
                not (type_ in SAFE_IFRAME_TYPES or
                     not remote_src)):
                self.err.warning(("testcases_markup_markuptester",
                                  "handle_starttag",
                                  "iframe_type_unsafe"),
                                 "iframe/browser missing 'type' attribute",
                                 "All iframe and browser elements must have "
                                 "either a valid `type` attribute or a `src` "
                                 "attribute that points to a local file.",
                                 self.filename,
                                 line=self.line,
                                 context=self.context)
            elif ((not type_ or
                   type_ not in SAFE_IFRAME_TYPES) and
                  remote_src):
                self.err.warning(("testcases_markup_markuptester",
                                  "handle_starttag",
                                  "iframe_type_unsafe"),
                                 "Typeless iframes/browsers must be local.",
                                 "iframe and browser elements that lack a type "
                                 "attribute must always have src attributes "
                                 "that reference local resources.",
                                 self.filename,
                                 line=self.line,
                                 context=self.context)

        elif tag == "script" and self.extension == "xul":
            # Per the Addon Validator Spec (v2), scripts in XUL
            # must not be remote.

            src = None
            for attr in attrs:
                if attr[0].lower() == "src":
                    src = attr[1].lower()

            if src and not self._is_url_local(src):
                self.err.warning(("testcases_markup_markuptester",
                                  "handle_starttag",
                                  "banned_remote_scripts"),
                                 "Scripts must not be remote in XUL",
                                 "In XUL, <script> tags must not be referenced "
                                 "to script files that are hosted remotely.",
                                 self.filename,
                                 line=self.line,
                                 context=self.context)

        # Find CSS and JS attributes and handle their values like they
        # would otherwise be handled by the standard parser flow.
        for attr in attrs:
            attr_name, attr_value = attr[0].lower(), attr[1]

            if (attr_name == "xmlns:xbl" and
                attr_value == "http://www.mozilla.org/xbl"):
                self.xbl = True

            if (self.err.detected_type == PACKAGE_THEME and
                attr_value.startswith(("data:", "javascript:"))):

                self.err.warning(
                        err_id=("testcases_markup_markuptester",
                                "handle_starttag",
                                "theme_attr_prefix"),
                        warning="Attribute contains banned prefix",
                        description=["A mark element's attribute contains a "
                                     "prefix which is not allowed in themes.",
                                     "Attribute: %s" % attr_name],
                        filename=self.filename,
                        line=self.line,
                        context=self.context)


            if attr_name == "style":
                csstester.test_css_snippet(self.err,
                                           self.filename,
                                           attr_value,
                                           self.line)
            elif attr_name.startswith("on"):  # JS attribute
                # Warn about DOM mutation event handlers.
                if attr_name in DOM_MUTATION_HANDLERS:
                    self.err.warning(
                        err_id=("testcases_markup_markuptester",
                                "handle_starttag",
                                "dom_manipulation_handler"),
                        warning="DOM Mutation Events Prohibited",
                        description="DOM mutation events are flagged because "
                                    "of their deprecated status, as well as "
                                    "their extreme inefficiency. Consider "
                                    "using a different event.",
                        filename=self.filename,
                        line=self.line,
                        context=self.context)

                scripting.test_js_snippet(err=self.err,
                                          data=attr_value,
                                          filename=self.filename,
                                          line=self.line,
                                          context=self.context)

            elif (self.extension == "xul" and
                  attr_name in ("insertbefore", "insertafter") and
                  any((id in attr[1]) for id in ("menu_pageSource",
                                                 "menu_pageinspect",
                                                 "javascriptConsole",
                                                 "webConsole"))):
                self.err.notice(
                    err_id=("testcases_markup_markuptester",
                            "handle_starttag",
                            "incompatible_menu_items"),
                    notice="Menu item has been moved",
                    description="Your add-on has an overlay that uses the "
                                "insertbefore or insertafter attribute "
                                "pointing to menuitems that have been moved "
                                "to a different menu item. Your overlay items "
                                "may appear in unexpected locations because "
                                "of this. See "
                        "https://bugzilla.mozilla.org/show_bug.cgi?id=653221"
                                " for more information.",
                    filename=self.filename,
                    line=self.line,
                    context=self.context,
                    for_appversions={'{ec8030f7-c20a-464f-9b0e-13a3a9e97384}':
                                         versions_after("firefox", "6.0a1")},
                    compatibility_type="warning")


        # When the dev forgets their <!-- --> on a script tag, bad
        # things happen.
        if "script" in self.xml_state and tag != "script":
            self._save_to_buffer("<" + tag + self._format_args(attrs) + ">")
            return

        self.xml_state.append(orig_tag)
        self.xml_line_stack.append(self.line)
        self.xml_buffer.append(unicode(""))

    def handle_endtag(self, tag):

        tag = tag.lower()
        if tag == "xul:script":
            tag = "script"

        if DEBUG:  # pragma: no cover
            print "E: ", tag, self.xml_state

        if not self.xml_state:
            if "closing_tags" in self.reported or not self.strict:
                if DEBUG:
                    print "Unstrict; extra closing tags ------"
                return
            self.err.warning(("testcases_markup_markuptester",
                              "handle_endtag",
                              "extra_closing_tags"),
                             "Markup parsing error",
                             "The markup file has more closing tags than it "
                             "has opening tags.",
                             self.filename,
                             line=self.line,
                             context=self.context,
                             tier=2)
            self.reported.add("closing_tags")
            if DEBUG:  # pragma: no cover
                print "Too many closing tags ------"
            return

{'fraction_non_alphanumeric': 0.08147718586076917, 'fraction_numerical': 0.003953923610195851, 'mean_word_length': 1.1312285826638953, 'pattern_counts': {'":': 20, '<': 35, '<?xml version=': 0, '>': 37, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 28}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 188, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5205925', 'n_tokens_mistral': 9248, 'n_tokens_neox': 8629, 'n_words': 2314}