<commit_before>import sys
import os
import fastchardet
import json
import fnmatch
from StringIO import StringIO

from validator import decorator
from validator.chromemanifest import ChromeManifest
from validator.xpi import XPIManager
from validator.constants import *

import validator.testcases.l10n.dtd as dtd
import validator.testcases.l10n.properties as properties


# The threshold that determines the number of entities that must not be
# missing from the package.
L10N_THRESHOLD = 0.35
L10N_SIMILAR_THRESHOLD = 0.9 # For en_US/en_GB kind of stuff

# Only warn about unchanged entities longer than this number of characters.
L10N_LENGTH_THRESHOLD = 3

# To avoid noise, this value ensures that the percent of unchanged entities
# is not inflated due to small numbers of entities.
L10N_MIN_ENTITIES = 18

LOCALE_CACHE = {}

def _get_locales(err, xpi_package):
    "Returns a list of locales from the chrome.manifest file."
    
    # Retrieve the chrome.manifest if it's cached.
    if err is not None and \
       err.get_resource("chrome.manifest"): # pragma: no cover
        chrome = err.get_resource("chrome.manifest")
    else:
        chrome_data = xpi_package.read("chrome.manifest")
        chrome = ChromeManifest(chrome_data)
        if err is not None:
            err.save_resource("chrome.manifest", chrome)
        
    pack_locales = chrome.get_triples("locale")
    locales = {}
    # Find all of the locales referenced in the chrome.manifest file.
    for locale in pack_locales:
        locale_jar = locale["object"].split()

        location = locale_jar[-1]
        if not location.startswith("jar:"):
            continue
        full_location = location[4:].split("!")
        locale_desc = {"predicate": locale["predicate"],
                       "path": full_location[0],
                       "target": full_location[1],
                       "name": locale_jar[0]}
        locale_name = "%s:%s" % (locale["predicate"], locale_jar[0])
        if locale_name not in locales:
            locales[locale_name] = locale_desc
    
    return locales

def _get_locale_manager(err, addon, path, files, no_cache=False):
    "Returns the XPIManager object for a locale"

    if path in LOCALE_CACHE and not no_cache:
        return LOCALE_CACHE[path]

    if path not in files:
        err.warning(("testcases_l10ncompleteness",
                     "_get_locale_manager",
                     "manager_absent"),
                    "Listed locale does not exist",
                    ["A locale JAR is listed in chrome.manifest, but it could "
                     "not be located. Check the spelling and capitalization "
                     "of the path.",
                     "Missing JAR: %s" % path],
                    filename="chrome.manifest")
        return None
    jar = StringIO(addon.read(path))
    locale = XPIManager(jar, path)

    if not no_cache:
        LOCALE_CACHE[path] = locale
    return locale

@decorator.register_test(tier=4)
def test_xpi(err, package_contents, xpi_package):
    """Tests an XPI (or JAR, really) for L10n completeness"""
    
    # Skip over incompatible (or unnecessary) package types.
    if err.detected_type not in (PACKAGE_EXTENSION,
                                 PACKAGE_THEME) or \
       err.is_nested_package():
        # NOTE : Should we also do this with PACKAGE_MULTI?
        return None
    
    # Don't even both with the test(s) if there's no chrome.manifest.
    if "chrome.manifest" not in package_contents:
        return None
    
    locales = _get_locales(err, xpi_package);
    
    # We need at least a reference and a target.
    if len(locales) < 2:
        return
    
    ref_name = "en-US"
    # Fall back on whatever comes first.
    if ref_name not in locales:
        ref_name = locales.keys()[0]

    reference = locales[ref_name]
    reference_locale = _get_locale_manager(err,
                                           xpi_package,
                                           reference["path"],
                                           package_contents)
    # Loop through the locales and test the valid ones.
    for name, locale in locales.items():
        # Ignore the reference locale
        if locale["name"] == ref_name:
            continue
        
        target_locale = _get_locale_manager(err,
                                            xpi_package,
                                            locale["path"],
                                            package_contents)
        if target_locale is None:
            continue
        split_target = locale["name"].split("-")
        
        # Isolate each of the target locales' results.
        results = _compare_packages(reference_locale,
                                    target_locale,
                                    reference["target"],
                                    locale["target"])
        _aggregate_results(err,
                           results,
                           locale,
                           ref_name.startswith(split_target[0]))

    # Clear the cache at the end of the test
    L10N_CACHE = {}

@decorator.register_test(tier=4, expected_type=PACKAGE_LANGPACK)
def test_lp_xpi(err, package_contents, xpi_package):
    "Tests a language pack for L10n completeness"
    
    # Don't even both with the test(s) if there's no chrome.manifest.
    if "chrome.manifest" not in package_contents:
        return None<commit_msg>Fix default locale selection and handling<commit_after>import sys
import os
import fastchardet
import json
import fnmatch
from StringIO import StringIO

from validator import decorator
from validator.chromemanifest import ChromeManifest
from validator.xpi import XPIManager
from validator.constants import *

import validator.testcases.l10n.dtd as dtd
import validator.testcases.l10n.properties as properties


# The threshold that determines the number of entities that must not be
# missing from the package.
L10N_THRESHOLD = 0.35
L10N_SIMILAR_THRESHOLD = 0.9 # For en_US/en_GB kind of stuff

# Only warn about unchanged entities longer than this number of characters.
L10N_LENGTH_THRESHOLD = 3

# To avoid noise, this value ensures that the percent of unchanged entities
# is not inflated due to small numbers of entities.
L10N_MIN_ENTITIES = 18

LOCALE_CACHE = {}

def _get_locales(err, xpi_package):
    "Returns a list of locales from the chrome.manifest file."
    
    # Retrieve the chrome.manifest if it's cached.
    if err is not None and \
       err.get_resource("chrome.manifest"): # pragma: no cover
        chrome = err.get_resource("chrome.manifest")
    else:
        chrome_data = xpi_package.read("chrome.manifest")
        chrome = ChromeManifest(chrome_data)
        if err is not None:
            err.save_resource("chrome.manifest", chrome)
        
    pack_locales = chrome.get_triples("locale")
    locales = {}
    # Find all of the locales referenced in the chrome.manifest file.
    for locale in pack_locales:
        locale_jar = locale["object"].split()

        location = locale_jar[-1]
        if not location.startswith("jar:"):
            continue
        full_location = location[4:].split("!")
        locale_desc = {"predicate": locale["predicate"],
                       "path": full_location[0],
                       "target": full_location[1],
                       "name": locale_jar[0]}
        locale_name = "%s:%s" % (locale["predicate"], locale_jar[0])
        if locale_name not in locales:
            locales[locale_name] = locale_desc
    
    return locales

def _get_locale_manager(err, addon, path, files, no_cache=False):
    "Returns the XPIManager object for a locale"

    if path in LOCALE_CACHE and not no_cache:
        return LOCALE_CACHE[path]

    if path not in files:
        err.warning(("testcases_l10ncompleteness",
                     "_get_locale_manager",
                     "manager_absent"),
                    "Listed locale does not exist",
                    ["A locale JAR is listed in chrome.manifest, but it could "
                     "not be located. Check the spelling and capitalization "
                     "of the path.",
                     "Missing JAR: %s" % path],
                    filename="chrome.manifest")
        return None
    jar = StringIO(addon.read(path))
    locale = XPIManager(jar, path)

    if not no_cache:
        LOCALE_CACHE[path] = locale
    return locale

@decorator.register_test(tier=4)
def test_xpi(err, package_contents, xpi_package):
    """Tests an XPI (or JAR, really) for L10n completeness"""
    
    # Skip over incompatible (or unnecessary) package types.
    if err.detected_type not in (PACKAGE_EXTENSION,
                                 PACKAGE_THEME) or \
       err.is_nested_package():
        # NOTE : Should we also do this with PACKAGE_MULTI?
        return None
    
    # Don't even both with the test(s) if there's no chrome.manifest.
    if "chrome.manifest" not in package_contents:
        return None
    
    locales = _get_locales(err, xpi_package);
    
    # We need at least a reference and a target.
    if len(locales) < 2:
        return
    
    # Use the first locale by default
    ref_name = locales.keys()[0]
    # Try to find en-US, as this is where the majority of users is
    for name, locale in locales.items():
        if locale["name"] == 'en-US':
            ref_name = name
            break

    reference = locales[ref_name]
    reference_locale = _get_locale_manager(err,
                                           xpi_package,
                                           reference["path"],
                                           package_contents)
    # Loop through the locales and test the valid ones.
    for name, locale in locales.items():
        # Ignore the reference locale
        if name == ref_name:
            continue
        
        target_locale = _get_locale_manager(err,
                                            xpi_package,
                                            locale["path"],
                                            package_contents)
        if target_locale is None:
            continue
        split_target = locale["name"].split("-")
        
        # Isolate each of the target locales' results.
        results = _compare_packages(reference_locale,
                                    target_locale,
                                    reference["target"],
                                    locale["target"])
        _aggregate_results(err,
                           results,
                           locale,
                           ref_name.startswith(split_target[0]))

    # Clear the cache at the end of the test
    L10N_CACHE = {}

@decorator.register_test(tier=4, expected_type=PACKAGE_LANGPACK)
def test_lp_xpi(err, package_contents, xpi_package):
    "Tests a language pack for L10n completeness"
    
    # Don't even both with the test(s) if there's no chrome.manifest.
    if "chrome.manifest" not in package_contents:
        return None
{'fraction_non_alphanumeric': 0.07603770164944716, 'fraction_numerical': 0.0070690592713431215, 'mean_word_length': 1.9840454299621417, 'pattern_counts': {'":': 8, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4612806', 'n_tokens_mistral': 3094, 'n_tokens_neox': 2935, 'n_words': 919}