<commit_before>import fnmatch

import validator.decorator as decorator
from validator.decorator import versions_after
from validator.constants import *

# Detect blacklisted files based on their extension.
blacklisted_extensions = ("dll", "exe", "dylib", "so",
                          "sh", "class", "swf")

blacklisted_magic_numbers = (
        (0x4d, 0x5a),  # EXE/DLL
        (0x5a, 0x4d),  # Alternative for EXE/DLL
        (0x7f, 0x45, 0x4c, 0x46),  # UNIX elf
        (0x23, 0x21),  # Shebang (shell script)
        (0xca, 0xfe, 0xba, 0xbe),  # Java + Mach-O (dylib)
        (0xca, 0xfe, 0xd0, 0x0d),  # Java (packed)
        (0xfe, 0xed, 0xfa, 0xce),  # Mach-O
        (0x46, 0x57, 0x53),  # Uncompressed SWF
        (0x43, 0x57, 0x53),  # ZLIB compressed SWF
)


def test_unknown_file(err, filename):
    "Tests some sketchy files that require silly code."

    # Extract the file path and the name from the filename
    path = filename.split("/")
    name = path.pop()

    if name == "chromelist.txt":
        err.notice(("testcases_packagelayout",
                    "test_unknown_file",
                    "deprecated_file"),
                   "Extension contains a deprecated file",
                   "The file \"%s\" is no longer supported by any modern "
                   "Mozilla product." % filename,
                   filename)
        return True


@decorator.register_test(tier=2, expected_type=PACKAGE_EXTENSION)
def test_npapi(err, package_contents=None, xpi_package=None):
    "Detects NPAPI plugins in extensions and flags them"

    matches = [fn for
               fn in
               package_contents if
               fnmatch.fnmatch(fn, "plugins/*.dll")]
    if any(matches):
        err.warning(("testcases_packagelayout",
                     "test_npapi",
                     "found_dll"),
                    "NPAPI plugin(s) detected in extension",
                    ["NPAPI plugins are generally disallowed in extensions. "
                     "The only approved channel for plugins is the Plugin "
                     "Finder Service.",
                     "Detected the following files: %s" % ", ".join(matches)],
                    "plugins/")


@decorator.register_test(tier=1)
def test_blacklisted_files(err, package_contents=None, xpi_package=None):
    "Detects blacklisted files and extensions."

    for name, file_ in package_contents.items():
        # Simple test to ensure that the extension isn't blacklisted
        extension = file_["extension"]
        if extension in blacklisted_extensions:
            # Note that there is a binary extension in the metadata
            err.metadata["contains_binary_extension"] = True
            err.warning(
                err_id=("testcases_packagelayout",
                        "test_blacklisted_files",
                        "disallowed_extension"),
                warning="Flagged file extension found",
                description=["The file \"%s\" has a flagged file extension." %
                                 name,
                             "The extension of this file is flagged because "
                             "it usually identifies binary components. Please "
                             "see "
                             "http://addons.mozilla.org/developers/docs/"
                                 "policies/reviews#section-binary"
                             " for more information on the binary content "
                             "review process."],
                filename=name)
            continue

        # Perform a deep inspection to detect magic numbers for known binary
        # and executable file types.
        zip = xpi_package.zf.open(name)
        bytes = tuple([ord(x) for x in zip.read(4)])  # Longest is 4 bytes
        if [x for x in blacklisted_magic_numbers if bytes[0:len(x)] == x]:
            # Note that there is binary content in the metadata
            err.metadata["contains_binary_content"] = True
            err.warning(
                err_id=("testcases_packagelayout",
                        "test_blacklisted_files",
                        "disallowed_file_type"),
                warning="Flagged file type found",
                description=["A file was found to contain flagged content "
                             "(i.e.: executable data, potentially "
                             "unauthorized scripts, etc.).",
                             u"The file \"%s\" contains flagged content" %
                                 name],
                filename=name)


@decorator.register_test(
        tier=1,
        versions={"{ec8030f7-c20a-464f-9b0e-13a3a9e97384}":<commit_msg>Changed compatibility_binary testcase to tier 5 (bug 658565)<commit_after>import fnmatch

import validator.decorator as decorator
from validator.decorator import versions_after
from validator.constants import *

# Detect blacklisted files based on their extension.
blacklisted_extensions = ("dll", "exe", "dylib", "so",
                          "sh", "class", "swf")

blacklisted_magic_numbers = (
        (0x4d, 0x5a),  # EXE/DLL
        (0x5a, 0x4d),  # Alternative for EXE/DLL
        (0x7f, 0x45, 0x4c, 0x46),  # UNIX elf
        (0x23, 0x21),  # Shebang (shell script)
        (0xca, 0xfe, 0xba, 0xbe),  # Java + Mach-O (dylib)
        (0xca, 0xfe, 0xd0, 0x0d),  # Java (packed)
        (0xfe, 0xed, 0xfa, 0xce),  # Mach-O
        (0x46, 0x57, 0x53),  # Uncompressed SWF
        (0x43, 0x57, 0x53),  # ZLIB compressed SWF
)


def test_unknown_file(err, filename):
    "Tests some sketchy files that require silly code."

    # Extract the file path and the name from the filename
    path = filename.split("/")
    name = path.pop()

    if name == "chromelist.txt":
        err.notice(("testcases_packagelayout",
                    "test_unknown_file",
                    "deprecated_file"),
                   "Extension contains a deprecated file",
                   "The file \"%s\" is no longer supported by any modern "
                   "Mozilla product." % filename,
                   filename)
        return True


@decorator.register_test(tier=2, expected_type=PACKAGE_EXTENSION)
def test_npapi(err, package_contents=None, xpi_package=None):
    "Detects NPAPI plugins in extensions and flags them"

    matches = [fn for
               fn in
               package_contents if
               fnmatch.fnmatch(fn, "plugins/*.dll")]
    if any(matches):
        err.warning(("testcases_packagelayout",
                     "test_npapi",
                     "found_dll"),
                    "NPAPI plugin(s) detected in extension",
                    ["NPAPI plugins are generally disallowed in extensions. "
                     "The only approved channel for plugins is the Plugin "
                     "Finder Service.",
                     "Detected the following files: %s" % ", ".join(matches)],
                    "plugins/")


@decorator.register_test(tier=1)
def test_blacklisted_files(err, package_contents=None, xpi_package=None):
    "Detects blacklisted files and extensions."

    for name, file_ in package_contents.items():
        # Simple test to ensure that the extension isn't blacklisted
        extension = file_["extension"]
        if extension in blacklisted_extensions:
            # Note that there is a binary extension in the metadata
            err.metadata["contains_binary_extension"] = True
            err.warning(
                err_id=("testcases_packagelayout",
                        "test_blacklisted_files",
                        "disallowed_extension"),
                warning="Flagged file extension found",
                description=["The file \"%s\" has a flagged file extension." %
                                 name,
                             "The extension of this file is flagged because "
                             "it usually identifies binary components. Please "
                             "see "
                             "http://addons.mozilla.org/developers/docs/"
                                 "policies/reviews#section-binary"
                             " for more information on the binary content "
                             "review process."],
                filename=name)
            continue

        # Perform a deep inspection to detect magic numbers for known binary
        # and executable file types.
        zip = xpi_package.zf.open(name)
        bytes = tuple([ord(x) for x in zip.read(4)])  # Longest is 4 bytes
        if [x for x in blacklisted_magic_numbers if bytes[0:len(x)] == x]:
            # Note that there is binary content in the metadata
            err.metadata["contains_binary_content"] = True
            err.warning(
                err_id=("testcases_packagelayout",
                        "test_blacklisted_files",
                        "disallowed_file_type"),
                warning="Flagged file type found",
                description=["A file was found to contain flagged content "
                             "(i.e.: executable data, potentially "
                             "unauthorized scripts, etc.).",
                             u"The file \"%s\" contains flagged content" %
                                 name],
                filename=name)


@decorator.register_test(
        tier=5,
        versions={"{ec8030f7-c20a-464f-9b0e-13a3a9e97384}":
{'fraction_non_alphanumeric': 0.09115110211905016, 'fraction_numerical': 0.018421893302097755, 'mean_word_length': 1.714450867052023, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5308360', 'n_tokens_mistral': 2678, 'n_tokens_neox': 2479, 'n_words': 716}