<commit_before>from collections import defaultdict
from datetime import timedelta

from django.apps import apps
from django.conf import settings
from django.core.checks import Error, Info, Warning, register as register_check
from django.core.exceptions import ImproperlyConfigured
from django.db.utils import OperationalError, ProgrammingError

import requests.exceptions

from normandy.recipes import signing, geolocation


INFO_COULD_NOT_RETRIEVE_ACTIONS = "normandy.recipes.I001"
INFO_COULD_NOT_RETRIEVE_RECIPES = "normandy.recipes.I002"
INFO_COULD_NOT_RETRIEVE_SIGNATURES = "normandy.recipes.I003"
WARNING_COULD_NOT_CHECK_SIGNATURES = "normandy.recipes.W001"
ERROR_MISMATCHED_ACTION_HASH = "normandy.recipes.E001"
ERROR_INVALID_RECIPE_SIGNATURE = "normandy.recipes.E002"
ERROR_BAD_SIGNING_CERTIFICATE = "normandy.recipes.E003"
ERROR_INVALID_ACTION_SIGNATURE = "normandy.recipes.E004"
ERROR_COULD_NOT_VERIFY_CERTIFICATE = "normandy.recipes.E005"
ERROR_GEOIP_DB_NOT_AVAILABLE = "normandy.recipes.E006"
ERROR_GEOIP_DB_UNEXPECTED_RESULT = "normandy.recipes.E007"


def actions_have_consistent_hashes(app_configs, **kwargs):
    errors = []
    try:
        Action = apps.get_model("recipes", "Action")
        actions = list(Action.objects.filter(implementation__isnull=False))
    except (ProgrammingError, OperationalError, ImproperlyConfigured) as e:
        errors.append(Info(f"Could not retrieve actions: {e}", id=INFO_COULD_NOT_RETRIEVE_ACTIONS))
    else:
        for action in actions:
            if action.compute_implementation_hash() != action.implementation_hash:
                msg = "Action '{action}' (id={action.id}) has a mismatched hash".format(
                    action=action
                )
                errors.append(Error(msg, id=ERROR_MISMATCHED_ACTION_HASH))

    return errors


def recipe_signatures_are_correct(app_configs, **kwargs):
    errors = []
    try:
        Recipe = apps.get_model("recipes", "Recipe")
        # pre-fetch signatures, to avoid race condition with deleted signatures
        signed_recipes = list(Recipe.objects.exclude(signature=None).select_related("signature"))
    except (ProgrammingError, OperationalError, ImproperlyConfigured) as e:
        errors.append(Info(f"Could not retrieve recipes: {e}", id=INFO_COULD_NOT_RETRIEVE_RECIPES))
        return errors

    try:
        for recipe in signed_recipes:
            data = recipe.canonical_json()
            signature = recipe.signature.signature
            pubkey = recipe.signature.public_key
            try:
                signing.verify_signature_pubkey(data, signature, pubkey)
            except signing.BadSignature as e:
                msg = "Recipe '{recipe}' (id={recipe.id}) has a bad signature: {detail}".format(
                    recipe=recipe, detail=e.detail
                )
                errors.append(Error(msg, id=ERROR_INVALID_RECIPE_SIGNATURE))
    except (ProgrammingError, OperationalError, ImproperlyConfigured) as e:
        errors.append(
            Warning(f"Could not check signatures: {e}", id=WARNING_COULD_NOT_CHECK_SIGNATURES)
        )

    return errors


def action_signatures_are_correct(app_configs, **kwargs):
    errors = []
    try:
        Action = apps.get_model("recipes", "Action")
        # pre-fetch signatures, to avoid race condition with deleted signatures
        signed_actions = list(Action.objects.exclude(signature=None).select_related("signature"))
    except (ProgrammingError, OperationalError, ImproperlyConfigured) as e:
        msg = f"Could not retrieve actions: f{e}"
        errors.append(Info(msg, id=INFO_COULD_NOT_RETRIEVE_ACTIONS))
        return errors

    try:
        for action in signed_actions:
            data = action.canonical_json()
            signature = action.signature.signature
            pubkey = action.signature.public_key
            try:
                signing.verify_signature_pubkey(data, signature, pubkey)
            except signing.BadSignature as e:
                msg = f"Action '{action}' (id={action.id}) has a bad signature: {e.detail}"
                errors.append(Error(msg, id=ERROR_INVALID_ACTION_SIGNATURE))
    except (ProgrammingError, OperationalError, ImproperlyConfigured) as e:
        errors.append(
            Warning(f"Could not check signatures: {e}", id=WARNING_COULD_NOT_CHECK_SIGNATURES)
        )

    return errors


def signatures_use_good_certificates(app_configs, **kwargs):
    errors = []
    expire_early = None<commit_msg>Verify with x5u preferentially to pubkey<commit_after>from collections import defaultdict
from datetime import timedelta

from django.apps import apps
from django.conf import settings
from django.core.checks import Error, Info, Warning, register as register_check
from django.core.exceptions import ImproperlyConfigured
from django.db.utils import OperationalError, ProgrammingError

import requests.exceptions

from normandy.recipes import signing, geolocation


INFO_COULD_NOT_RETRIEVE_ACTIONS = "normandy.recipes.I001"
INFO_COULD_NOT_RETRIEVE_RECIPES = "normandy.recipes.I002"
INFO_COULD_NOT_RETRIEVE_SIGNATURES = "normandy.recipes.I003"
WARNING_COULD_NOT_CHECK_SIGNATURES = "normandy.recipes.W001"
ERROR_MISMATCHED_ACTION_HASH = "normandy.recipes.E001"
ERROR_INVALID_RECIPE_SIGNATURE = "normandy.recipes.E002"
ERROR_BAD_SIGNING_CERTIFICATE = "normandy.recipes.E003"
ERROR_INVALID_ACTION_SIGNATURE = "normandy.recipes.E004"
ERROR_COULD_NOT_VERIFY_CERTIFICATE = "normandy.recipes.E005"
ERROR_GEOIP_DB_NOT_AVAILABLE = "normandy.recipes.E006"
ERROR_GEOIP_DB_UNEXPECTED_RESULT = "normandy.recipes.E007"


def actions_have_consistent_hashes(app_configs, **kwargs):
    errors = []
    try:
        Action = apps.get_model("recipes", "Action")
        actions = list(Action.objects.filter(implementation__isnull=False))
    except (ProgrammingError, OperationalError, ImproperlyConfigured) as e:
        errors.append(Info(f"Could not retrieve actions: {e}", id=INFO_COULD_NOT_RETRIEVE_ACTIONS))
    else:
        for action in actions:
            if action.compute_implementation_hash() != action.implementation_hash:
                msg = "Action '{action}' (id={action.id}) has a mismatched hash".format(
                    action=action
                )
                errors.append(Error(msg, id=ERROR_MISMATCHED_ACTION_HASH))

    return errors


def recipe_signatures_are_correct(app_configs, **kwargs):
    errors = []
    try:
        Recipe = apps.get_model("recipes", "Recipe")
        # pre-fetch signatures, to avoid race condition with deleted signatures
        signed_recipes = list(Recipe.objects.exclude(signature=None).select_related("signature"))
    except (ProgrammingError, OperationalError, ImproperlyConfigured) as e:
        errors.append(Info(f"Could not retrieve recipes: {e}", id=INFO_COULD_NOT_RETRIEVE_RECIPES))
        return errors

    try:
        for recipe in signed_recipes:
            data = recipe.canonical_json()
            signature = recipe.signature.signature
            pubkey = recipe.signature.public_key
            x5u = recipe.signature.x5u
            try:
                if x5u:
                    signing.verify_signature_x5u(data, signature, x5u)
                else:
                    signing.verify_signature_pubkey(data, signature, pubkey)
            except signing.BadSignature as e:
                msg = "Recipe '{recipe}' (id={recipe.id}) has a bad signature: {detail}".format(
                    recipe=recipe, detail=e.detail
                )
                errors.append(Error(msg, id=ERROR_INVALID_RECIPE_SIGNATURE))
    except (ProgrammingError, OperationalError, ImproperlyConfigured) as e:
        errors.append(
            Warning(f"Could not check signatures: {e}", id=WARNING_COULD_NOT_CHECK_SIGNATURES)
        )

    return errors


def action_signatures_are_correct(app_configs, **kwargs):
    errors = []
    try:
        Action = apps.get_model("recipes", "Action")
        # pre-fetch signatures, to avoid race condition with deleted signatures
        signed_actions = list(Action.objects.exclude(signature=None).select_related("signature"))
    except (ProgrammingError, OperationalError, ImproperlyConfigured) as e:
        msg = f"Could not retrieve actions: f{e}"
        errors.append(Info(msg, id=INFO_COULD_NOT_RETRIEVE_ACTIONS))
        return errors

    try:
        for action in signed_actions:
            data = action.canonical_json()
            signature = action.signature.signature
            pubkey = action.signature.public_key
            x5u = action.signature.x5u
            try:
                if x5u:
                    signing.verify_signature_x5u(data, signature, x5u)
                else:
                    signing.verify_signature_pubkey(data, signature, pubkey)
            except signing.BadSignature as e:
                msg = f"Action '{action}' (id={action.id}) has a bad signature: {e.detail}"
                errors.append(Error(msg, id=ERROR_INVALID_ACTION_SIGNATURE))
    except (ProgrammingError, OperationalError, ImproperlyConfigured) as e:
        errors.append(
            Warning(f"Could not check signatures: {e}", id=WARNING_COULD_NOT_CHECK_SIGNATURES)
        )

    return errors


def signatures_use_good_certificates(app_configs, **kwargs):
    errors = []
    expire_early = None
{'fraction_non_alphanumeric': 0.08648996673462818, 'fraction_numerical': 0.00826268912973495, 'mean_word_length': 3.2713107241063244, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '328393', 'n_tokens_mistral': 2888, 'n_tokens_neox': 2749, 'n_words': 564}